1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

This VM requires Nvidia drivers to function correctly.   Installation takes ~1 minute.
Would you like to install the Nvidia driver? [y/n] y
Installing Nvidia driver.
wait apt locks released
install linux headers: linux-headers-4.19.0-21-cloud-amd64
E: Could not get lock /var/lib/dpkg/lock-frontend - open (11: Resource temporarily unavailable)
E: Unable to acquire the dpkg frontend lock (/var/lib/dpkg/lock-frontend), is another process using it?
DRIVER_VERSION: 510.47.03
Downloading driver from GCS location and install: 
gs://nvidia-drivers-us-public/tesla/510.47.03/NVIDIA-Linux-x86_64-510.47.03.run

Verifying archive integrity... OK
Uncompressing NVIDIA Accelerated Graphics Driver for Linux-x86_64 510.47.03...[...]..

WARNING: The nvidia-drm module will not be installed.
 As a result, DRM-KMS will not function with this installation of the NVIDIA driver.


WARNING: nvidia-installer was forced to guess the X library path 
'/usr/lib64' and X module path '/usr/lib64/xorg/modules';
these paths were not queryable from the system.  If X fails to find the NVIDIA X driver module, please install the
         `pkg-config` utility and the X.Org SDK/development package for your distribution and reinstall the driver.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

apt update
apt install -y vim curl wget git build-essential 

curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py \
--output install_gpu_driver.py

python3 install_gpu_driver.py

useradd -s /bin/bash codeformer -m 

su - codeformer << EOF
git clone https://github.com/sczhou/CodeFormer.git
cd CodeFormer

wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
bash ./Anaconda3-2022.10-Linux-x86_64.sh -b -p \$HOME/anaconda3
eval "\$(\$HOME/anaconda3/bin/conda shell.bash hook)"
echo 'eval "\$(\$HOME/anaconda3/bin/conda shell.bash hook)' /home/codeformer/.bashrc
conda create -n codeformer python=3.8 -y
conda activate codeformer

pip3 install -r requirements.txt
pip3 install basicsr
python basicsr/setup.py develop

python scripts/download_pretrained_models.py facelib
python scripts/download_pretrained_models.py CodeFormer
EOF

1

gcloud compute images list --project deeplearning-platform-release

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

{
  "builders": [
    {
      "type": "googlecompute",
      "project_id": "face-reconstruction-api",
      "ssh_username": "packer",
      "zone": "europe-west4-a",
      "source_image": "tf-latest-gpu-v20221107-debian-10",
      "source_image_project_id": "deeplearning-platform-release",
      "on_host_maintenance": "TERMINATE",
      "accelerator_type": "projects/face-reconstruction-api/zones/europe-west4-a/acceleratorTypes/nvidia-tesla-t4",
      "accelerator_count": "1",
      "machine_type": "n1-standard-4",
      "disk_size": "50",
      "image_name": "codeformer-{{timestamp}}",
      "image_storage_locations": "europe-west4" 
     }
  ],
  "provisioners": [
    {
      "type": "shell",
      "scripts": ["scripts/update_and_driver_install.sh",
	"scripts/install_codeformer_in_user_home.sh"
      ]
    }
  ]
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

gcloud compute accelerator-types describe --zone europe-west4-a nvidia-tesla-t4
creationTimestamp: '1969-12-31T16:00:00.000-08:00'
description: NVIDIA T4
id: '10019'
kind: compute#acceleratorType
maximumCardsPerInstance: 4
name: nvidia-tesla-t4
selfLink: https://www.googleapis.com/compute/v1/projects/face-reconstruction-api/
zones/europe-west4-a/acceleratorTypes/nvidia-tesla-t4
zone: https://www.googleapis.com/compute/v1/projects/face-reconstruction-api/zones/europe-west4-a

1
2
3
4
5
6
7

scripts/update_and_driver_install.sh

set -x
echo "wait 4 minutes for gcp image to finish to auto update"
sleep 240
sudo apt update && sudo apt upgrade
sudo apt install -y vim curl wget git build-essential software-properties-common

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

install_codeformer_in_user_home.sh

sudo useradd -m codeformer

sudo su - codeformer << EOF
set -x
cd /home/codeformer
wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
bash ./Anaconda3-2022.10-Linux-x86_64.sh -b -p "/home/codeformer/anaconda3"

git clone https://github.com/sczhou/CodeFormer.git
cd CodeFormer

eval "\$(/home/codeformer/anaconda3/bin/conda shell.bash hook)"
echo 'eval "\$(/home/codeformer/anaconda3/bin/conda shell.bash hook)"' >> /home/codeformer/.bashrc

source "/home/codeformer/anaconda3/etc/profile.d/conda.sh"
eval "\$(/home/codeformer/anaconda3/bin/conda shell.bash hook)"

conda create -n codeformer python=3.8 -y
conda activate codeformer

curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py \
--output install_gpu_driver.py

python3 install_gpu_driver.py

pip install -r requirements.txt
pip install basicsr
python basicsr/setup.py develop
python scripts/download_pretrained_models.py facelib
python scripts/download_pretrained_models.py CodeFormer
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

#!/bin/bash
set -x

echo "copy deamon python deamon codeformer script" 
echo -e "${codeformer_loader_daemon}"|base64 -d >> /home/codeformer/CodeFormer/codeformer_loader_daemon.py
/home/codeformer/CodeFormer/codeformer_loader_daemon.py

echo "copy run_codeformer.sh"
echo "#!/bin/bash" > /home/codeformer/CodeFormer/run_deamon.sh
echo "export CLOUDRUN_URL='${cloudrun_destroy_url}'" >> /home/codeformer/CodeFormer/run_deamon.sh
echo -e "${run_codeformer}"|base64 -d >> /home/codeformer/CodeFormer/run_deamon.sh

echo "copy codeformer_process_images.sh"
echo -e "${codeformer_process_images}"|base64 -d > /home/codeformer/CodeFormer/codeformer_process_images.sh
chmod +x /home/codeformer/CodeFormer/codeformer_process_images.sh

chmod +x /home/codeformer/CodeFormer/run_deamon.sh
/home/codeformer/CodeFormer/run_deamon.sh

1
2
3
4
5

cd /home/codeformer/CodeFormer/
source /home/codeformer/anaconda3/etc/profile.d/conda.sh
conda activate codeformer
pip3 install google-cloud-storage==2.5.0 requests
python3 /home/codeformer/CodeFormer/codeformer_loader_daemon.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

#!/usr/bin/env python3
# coding:utf8

import sys
import os
import signal
import time
import subprocess
import glob
import logging
import requests
from google.cloud import storage

import urllib
import requests
import google.auth.transport.requests
import google.oauth2.id_token

logging.basicConfig(stream=sys.stdout, level=logging.INFO)

GCS_DATA_BUCKET = "facereconstruction-raw-images"
GCS_RESULT_BUCKET = "facereconstruction-result-images"
storage_client = storage.Client()
raw_image_input_path = "//home/codeformer/CodeFormer/inputs/whole_imgs/"
raw_image_result_path = "//home/codeformer/CodeFormer/results/whole_imgs_0.7/final_results/"
CLOUDRUN_URL = os.getenv("CLOUDRUN_URL")


class GracefulKiller:
    kill_now = False

    def __init__(self):
        signal.signal(signal.SIGINT, self.exit_gracefully)
        signal.signal(signal.SIGTERM, self.exit_gracefully)

    def exit_gracefully(self, *args):
        self.kill_now = True


def list_images_from_gcs():
    logging.info("get images from bucket")
    remote_images = storage_client.list_blobs(GCS_DATA_BUCKET)
    images_name = []
    for image in remote_images:
        images_name.append(image.name)
    return images_name


def clean_input_directory():
    files = glob.glob(raw_image_input_path + "*")
    for f in files:
        logging.info(f"remove file {f}")
        os.remove(f)


def copy_remote_images_on_fs(images):
    for image_name in images:
        logging.info(f"copy file {image_name} to code former input folder.")
        bucket = storage_client.bucket(GCS_DATA_BUCKET)
        blob = bucket.blob(image_name)
        blob.download_to_filename(raw_image_input_path + image_name)


def run_codeformer_command():
    logging.info("run codeformer command")
    os.system("./codeformer_process_images.sh")


def upload_result_to_remote():
    bucket_result = storage_client.bucket(GCS_RESULT_BUCKET)
    logging.info("upload result on result bucket")
    files = glob.glob(raw_image_result_path + "*")
    for file_result in files:
        logging.info(f"uploading file on remote {file_result}")
        blob = bucket_result.blob(os.path.basename(file_result))
        blob.upload_from_filename(file_result)


def delete_remote_images(images):
    logging.info("delete images raw bucket")
    bucket = storage_client.bucket(GCS_DATA_BUCKET)
    for image_name in images:
        blob = bucket.blob(image_name)
        blob.delete()


def get_auth_token_and_call_tf_destroy():
    tf_destroy_url=CLOUDRUN_URL + "tf_destroy"
    logging.info(f"calling destroy route {tf_destroy_url}")
    auth_req = google.auth.transport.requests.Request()
    id_token = google.oauth2.id_token.fetch_id_token(auth_req, CLOUDRUN_URL)
    
    headers = {"Authorization": f"Bearer {id_token}"}
    response = requests.get(tf_destroy_url, headers=headers)


if __name__ == "__main__":

    logging.info("deamon loaded !")
    killer = GracefulKiller()
    while not killer.kill_now:
        time.sleep(1)
        clean_input_directory()
        gcs_images = list_images_from_gcs()
        if not gcs_images:
            logging.info("no more images to process, exiting infinite loop.")
            break
        copy_remote_images_on_fs(gcs_images)
        run_codeformer_command()
        upload_result_to_remote()
        delete_remote_images(gcs_images)

    get_auth_token_and_call_tf_destroy()

    logging.info("End of the program. I was killed gracefully :)")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

locals {

  cloudrun_roles = [ 
"roles/storage.admin", # read write bucket
"roles/run.invoker", # call cloudrun tf destroy
]
}

resource "google_service_account" "compute_t4_sa" {
  account_id   = "t4-compute"
  display_name = "Service Account T4 compute"
}

resource "google_project_iam_member" "compute_admin" {
  project = var.project
  count   = length(local.cloudrun_roles)
  role    = local.cloudrun_roles[count.index]
  member  = "serviceAccount:${google_service_account.compute_t4_sa.email}"
}

data "template_file" "install_script" {
  template = file("${path.module}/cloud-init.sh")
  vars = {
    codeformer_loader_daemon    = base64encode(file("${path.module}/codeformer_loader_daemon.py"))
    codeformer_process_images   = base64encode(file("${path.module}/codeformer_process_images.sh"))
    run_codeformer              = base64encode(file("${path.module}/run_codeformer.sh"))
    cloudrun_destroy_url        = "${data.google_cloud_run_service.fastapi.status.0.url}/"
  }
}

resource "google_compute_instance" "default" {
  name         = "t4-compute"
  machine_type = "n1-standard-4"
  zone         = "europe-west4-a"

  boot_disk {
    initialize_params {
      image = "codeformer-1669048636"
    }
  }

  // Local SSD disk
  scratch_disk {
    interface = "SCSI"
  }

  network_interface {
    network = "default"

    access_config {
      // Ephemeral public IP
    }
  }

  scheduling {
    automatic_restart           = "false"
    preemptible                 = "true"
    on_host_maintenance         = "TERMINATE"
    instance_termination_action = "STOP"
    provisioning_model          = "SPOT"
  }

  guest_accelerator {
    type  = "nvidia-tesla-t4"
    count = "1"
  }
  metadata_startup_script = data.template_file.install_script.rendered

  service_account {
    email  = google_service_account.compute_t4_sa.email
    scopes = ["cloud-platform"]
  }
}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

import time
import os
import sys
import json
import logging
import urllib
import requests

import google.auth.transport.requests
import google.oauth2.id_token

from fastapi import FastAPI, File, UploadFile, Request
from fastapi.responses import HTMLResponse
from google.cloud import storage

import google.cloud.logging

client = google.cloud.logging.Client()
client.setup_logging()

GCS_DATA_BUCKET = os.getenv("GCS_DATA_BUCKET")
if not GCS_DATA_BUCKET:
    print("env var GCS_DATA_BUCKET is missing.")
    sys.exit(-1)

CLOUDRUN_PUBLIC_URL = os.getenv("CLOUDRUN_PUBLIC_URL")
if not CLOUDRUN_PUBLIC_URL:
    print("env var CLOUDRUN_PUBLIC_URL is missing.")
    sys.exit(-1)

client = storage.Client()
bucket = client.get_bucket(GCS_DATA_BUCKET)

app = FastAPI()

current_path = os.path.dirname(os.path.abspath(__name__))


@app.post("/uploadfiles/")
async def create_upload_files(
    request: Request,
    files: list[UploadFile] = File(description="Multiple files as UploadFile"),
):
    client_host = request.client.host

    exclude_file = []
    trigger_compute_creation = False

    for file in files:
        if file.filename.lower().endswith((".png", ".jpg", ".tiff", ".bmp", ".gif")):
            blob = bucket.blob(file.filename)
            with blob.open("wb") as f:
                f.write(file.file.read())
            trigger_compute_creation = True
        else:
            logging.info(f"skip file {file.filename} since extension is not supported")
            exclude_file.append(file.filename)

        if trigger_compute_creation:
            tf_apply_route = f"{CLOUDRUN_PUBLIC_URL}/tf_apply"
            logging.info(f"provisionning compute calling url {tf_apply_route}")
            os.chdir(current_path + "/001_gpu_compute/")
            logging.info("tf init and apply")
            os.system("terraform init")
            os.system("terraform apply --auto-approve")
            os.chdir(current_path)
            return "provision t4 compute launch !"
        else:
            return "you don't provide at least 1 valid image, skip compute creation ..."


@app.get("/")
async def main():
    return "c'est ok josé"

@app.get("/tf_destroy")
async def tf_destroy():
    os.chdir(current_path + "/001_gpu_compute/")
    os.system("terraform init")
    os.system("terraform destroy --auto-approve")
    os.chdir(current_path)
    return "c'est ok josé"