wpst_s1tiling_test.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7e988217",
   "metadata": {},
   "source": [
    "## This notebook serves as a test for wpst endpoints."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0e9b317f",
   "metadata": {},
   "source": [
    "In order to run this Notebook you need to make sure you have *copa_backend_url* , *s1tiling_cwl*, *client_id* and *url_token* set in /projects/.maap/maap.ini\n",
    "\n",
    "**copa_backend_url** : Url for copa bakend, use Url for the corresponding environement.\n",
    "\n",
    "**s1tiling_cwl** : Url of s3 location for the cwl of s1tiling algorithm. s1tiling_cwl = https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/s1-tiling-demo/workflow.cwl\n",
    "\n",
    "**client_id** : Client ID\n",
    "\n",
    "**url_token** : open id auth link\n",
    "\n",
    "If the file doesn't exist, please create it using terminal and your favorite editor."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9986d396",
   "metadata": {},
   "source": [
    "#### Import and conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43d3b75a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import sys\n",
    "import os\n",
    "import xml.etree.ElementTree as ET\n",
    "import time\n",
    "import sys\n",
    "import configparser\n",
    "import pandas as pd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0c68b487",
   "metadata": {},
   "source": [
    "We load auth configuration from file /projects/.maap/auth.ini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fd09ce3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load the file that contains auth values\n",
    "config = configparser.ConfigParser()\n",
    "config.read('/projects/.maap/auth.ini')\n",
    "\n",
    "#Retrieve auth values\n",
    "email = config['auth']['email']\n",
    "password = config['auth']['password']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b4518fef",
   "metadata": {},
   "source": [
    "Load configuration from /projects/.maap/auth.ini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bf1b52ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/projects/demo-scripts/wps_notebook/wpst_test/TestJobID.csv\n"
     ]
    }
   ],
   "source": [
    "#Load the file that contains maap values\n",
    "config = configparser.ConfigParser()\n",
    "config.read('/projects/.maap/maap.ini')\n",
    "\n",
    "#Retrieve maap values\n",
    "copa_backend_url = config['maap']['copa_backend_url']\n",
    "CLIENT_ID = config['maap']['client_id']\n",
    "url_token = config['maap']['url_token']\n",
    "\n",
    "#Location for tmp file for created process\n",
    "JobIdWD = '{}/TestJobID.csv'.format(os.path.abspath(os.getcwd()))\n",
    "print(JobIdWD)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b4a0542d",
   "metadata": {},
   "source": [
    "Get access_token using client_id, email and password from configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ad8e5d4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'access_token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI3dThmWGFxTnFGZHBCSzhLbDRTT1RDeFZodjE3ZUM5dnNZYnFDX2FXZXhZIn0.eyJleHAiOjE2ODM2NzEzMjcsImlhdCI6MTY4MzY0MjUyNywianRpIjoiZTg0MDE1MzUtMDcwZC00MzhjLTg3YmItNDFhM2QyNmY5MWFlIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLmRldi5lc2EtbWFhcC5vcmcvcmVhbG1zL21hYXAiLCJzdWIiOiI3ZmYwNGNlMy02NDRlLTQyOTctYTg2NC1lZWM3MWUyYmI0NWUiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJtYWFwLXRva2VuIiwic2Vzc2lvbl9zdGF0ZSI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCIsImFjciI6IjEiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJDYW1wYWluIEVkaXRvciIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6InByb2ZpbGUgZW1haWwiLCJzaWQiOiI5ODViMjY2Zi02YTgyLTQzNDUtOTBhNS1kN2M3ZjAzZGNiNjQiLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsIm5hbWUiOiJUZXN0IFRlc3QiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJ0ZXN0QHRlc3QudGVzdCIsImdpdmVuX25hbWUiOiJUZXN0IiwiZmFtaWx5X25hbWUiOiJUZXN0IiwiZW1haWwiOiJ0ZXN0QHRlc3QudGVzdCJ9.k0jMVH33fIQ05_9ok5-SbrZYUE-NHXEZvrtLKuJBF3BWWYzUzUfspKZ4CkA7s5R_4Lbpi1YiNdEuBIzSqR4gShmTA_HnG7sgJVe4R2hxUSH6197xr_GCY8hp4GPMaI4xw0NjPJ-gcXM9O8IJT6j8I0zhsAz_fg8mFO9M7lVv32Oa3QvVqNnspVOl8UA4zl2Hef9rUKDh1Jt--5bBoyxNJzS7Lwz4kslr8gaekFJDLoGho5w9lZVXENo4c-Vxovjmv2uvvUFP1vGmg9kNEBPk-fRc3a10qkBJf0lgCUKxg2v27qlqg_JXnCpdCKsmUbqv8slRCNufhtlZ8Gbp3KMRNQ', 'expires_in': 28800, 'refresh_expires_in': 36000, 'refresh_token': 'eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJjYTI3YWEzYS0zNTM5LTRiMzktODNhNS1kYWFmZDkzZTdmM2QifQ.eyJleHAiOjE2ODM2Nzg1MjcsImlhdCI6MTY4MzY0MjUyNywianRpIjoiZDA2NGQ3MjMtNjgwZC00NzYyLTg3ZTQtOTgzNmRjYWUxODUyIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLmRldi5lc2EtbWFhcC5vcmcvcmVhbG1zL21hYXAiLCJhdWQiOiJodHRwczovL2F1dGguZGV2LmVzYS1tYWFwLm9yZy9yZWFsbXMvbWFhcCIsInN1YiI6IjdmZjA0Y2UzLTY0NGUtNDI5Ny1hODY0LWVlYzcxZTJiYjQ1ZSIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJtYWFwLXRva2VuIiwic2Vzc2lvbl9zdGF0ZSI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCIsInNjb3BlIjoicHJvZmlsZSBlbWFpbCIsInNpZCI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCJ9.Wc3pXRAud6U6xAklFUKztJCwtfz0QVBPx1zljLQ9eT8', 'token_type': 'Bearer', 'not-before-policy': 0, 'session_state': '985b266f-6a82-4345-90a5-d7c7f03dcb64', 'scope': 'profile email'}\n"
     ]
    }
   ],
   "source": [
    "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"profile\"})\n",
    "data = json.loads(response.text)\n",
    "print(data)\n",
    "oauth_token = data['access_token']"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "33c2e6dc",
   "metadata": {},
   "source": [
    "#### WPST process deployement"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "120ba445",
   "metadata": {},
   "source": [
    "For this test we don't want to interfere with existant processes we already have. thus we will list the prcesses we have and check if s1tiling is deployed and get it's id, if not we deploy it.\n",
    "\n",
    "WPST end point give possibility to list all the processes deployed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d9e3ce0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"processes\":[]}'\n"
     ]
    }
   ],
   "source": [
    "# get deployed cwl workflow\n",
    "process = requests.get(copa_backend_url+'wpst/processes', headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "print(process.content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "67ec578e",
   "metadata": {},
   "source": [
    "In the next step we will make a loop on the previous response *process* payload in order to get a deployement id for 'wf-s1tiling-demo'. if found we save the id in *process_id* variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e79d0c10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "# get wf-s1tiling-demo id if exist\n",
    "process_already_deployed = None\n",
    "for item in json.loads(process.content).get(\"processes\"):\n",
    "    if item.get('title')== 'wf-s1tiling-demo':\n",
    "        process_already_deployed = item.get('id')\n",
    "print(process_already_deployed)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3c1c1f81",
   "metadata": {},
   "source": [
    "In the next step, if s1tiling is not already deployed, we read s1tiling cwl storage link from configuration and we call the deploy process endpoint using a post method to create a new s1-tiling deployement.\n",
    "\n",
    "At the end of this step we will have workflow_id containing our s1tiling id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "dd258b84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "b'{\"processSummary\":{\"id\":\"645a588a19eb877e986b73d2\",\"title\":\"wf-s1tiling-demo\",\"keywords\":null,\"owsContext\":null,\"metadata\":null,\"additionalParameters\":null,\"links\":null,\"version\":null,\"jobControlOptions\":null,\"outputTransmission\":null,\"processDescriptionURL\":\"https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/s1-tiling-demo/workflow.cwl\",\"abstract\":\"Launch S1-Tiling Algorithm\"}}'\n"
     ]
    }
   ],
   "source": [
    "#deploy wf-s1-tiling\n",
    "url_s1tiling_cwl = config['maap']['s1tiling_cwl']\n",
    "wfstorage =  {\"executionUnit\": [{\"href\": \"{}\".format(url_s1tiling_cwl)}]}\n",
    "if not process_already_deployed:\n",
    "    workflow = requests.post(\n",
    "                            copa_backend_url+'wpst/processes/', \n",
    "                            headers = {'Authorization': 'Bearer '+ oauth_token}, \n",
    "                            json=wfstorage)\n",
    "    print(workflow.status_code)\n",
    "    print(workflow.content)\n",
    "    workflow_id = json.loads(workflow.content).get('processSummary').get('id')\n",
    "else:\n",
    "    workflow_id = process_already_deployed"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7fa1b556",
   "metadata": {},
   "source": [
    "#### Launch o process and get status\n",
    "This mark the succes of deploy test, now we move on to testing launching a process and getting its status."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "bf7952e9",
   "metadata": {},
   "source": [
    "We can get a description of a given process, therefore we can check the inputs needed to luanch this process and other details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c7f7ec9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'id': 'input_s1_l1_grd', 'title': 'S1 GRD products folder', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string[]', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Folder containing input S1 L1 GRD product'}, {'id': 's_parallel_proc', 'title': 'Number of parallel process (eg S1 image)', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Number of parallel process (eg S1 image)'}, {'id': 's_ram_per_proc', 'title': 'RAM per process (Mb)', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'RAM Allowed per process in MB'}, {'id': 's_otb_thread', 'title': 'Number of thread in OTB', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Number of thread in otb'}, {'id': 's_light_mode', 'title': 'Run in light mode', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Run in light mode (True/False)'}, {'id': 'bucket_name', 'title': 'Stage-in source bucket', 'keywords': ['default: maap-scientific-data'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Name of the S3 bucket containing the input data'}, {'id': 'copy_dir_or_file', 'title': 'Stage-in mode (dir or file)', 'keywords': ['default: dir'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Flag indicating whether input data is a directory (value=dir) or a file or list of files (value=file)'}, {'id': 's3_destination', 'title': 'Stage-out target folder', 'keywords': ['default: maap-scientific-data/shared/TOBE_COMPLETED'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Bucket name + destination folder in which to transfer result data'}]\n"
     ]
    }
   ],
   "source": [
    "wf_description = requests.get(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token}) \n",
    "print(json.loads(wf_description.content).get('process').get('inputs'))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f1ad2196",
   "metadata": {},
   "source": [
    "Test that the description of the inputs provided from the backend match the awaited list, if any modification accured on the inputs for s1tiling algorithm it's imperative to add the right values in the list below. Therefore this test make sure we don't have any unwanted regression ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "de604442",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "# Launch s1 tiling wf\n",
    "awaited_imputs = [ \n",
    "    'Number of parallel process (eg S1 image)',\n",
    "    'RAM per process (Mb)',\n",
    "    'Number of thread in OTB',\n",
    "    'Run in light mode',\n",
    "    'Stage-in source bucket',\n",
    "    'Stage-in mode (dir or file)',\n",
    "    'Stage-out target folder',\n",
    "    'S1 GRD products folder',\n",
    "]\n",
    "print(len(awaited_imputs))\n",
    "for item in json.loads(wf_description.content).get('process').get('inputs'):\n",
    "    assert item.get('title') in awaited_imputs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1ffaf067",
   "metadata": {},
   "source": [
    "Creating payload for our request to launch an excution of the s1tiling process, the main focus on this part is to load values that the algorithm takes as input.\n",
    "\n",
    "Note that we give data correlated to the input id and not input title. the inputs titles above correspond respectively to the ids below ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3cdfc486",
   "metadata": {},
   "outputs": [],
   "source": [
    "payload = {\n",
    "  \"inputs\": [\n",
    "    {\n",
    "      \"id\": \"s_parallel_proc\",\n",
    "      \"data\": \"1\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"s_ram_per_proc\",\n",
    "      \"data\": \"8096\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"s_otb_thread\",\n",
    "      \"data\": \"1\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"s_light_mode\",\n",
    "      \"data\": \"True\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"bucket_name\",\n",
    "      \"data\": \"maap-scientific-data\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"copy_dir_or_file\",\n",
    "      \"data\": \"dir\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"s3_destination\",\n",
    "      \"data\": \"maap-scientific-data/adu/test\",\n",
    "      \"href\": \"\"\n",
    "    },\n",
    "    {\n",
    "      \"id\": \"input_s1_l1_grd\",\n",
    "      \"data\": \"shared/adu/S1_GRD/S2tiles/35SPB\",\n",
    "      \"href\": \"\"\n",
    "    }\n",
    "  ],\n",
    "  \"outputs\": [],\n",
    "  \"mode\": \"ASYNC\",\n",
    "  \"response\": \"RAW\"\n",
    "}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ccada0dc",
   "metadata": {},
   "source": [
    "Run the process using the above payload."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f683815d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [201]>\n"
     ]
    }
   ],
   "source": [
    "wf_exec = requests.post(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), json=payload, headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "job_id = json.loads(wf_exec.content).get('jobId')\n",
    "print(wf_exec)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "567b516c",
   "metadata": {},
   "source": [
    "Write jobId in TestJobID.csv to save the jobs launched in this test, to be removed all at the end. Cn case of a falure of tests after this step, the id will always be in TestJobID.csv and when we reach the step 16 we can remove them all at once."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c7177b32",
   "metadata": {},
   "outputs": [],
   "source": [
    "jobIds = []\n",
    "if os.path.exists(JobIdWD):\n",
    "    df2 = pd.read_csv(JobIdWD)\n",
    "    for item in df2.values:\n",
    "        jobIds.append(str(item[0]))\n",
    "\n",
    "if job_id not in jobIds:\n",
    "    jobIds.append(job_id)\n",
    "df = pd.DataFrame({'jobID':jobIds})\n",
    "df.to_csv(JobIdWD, index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "03c909c6",
   "metadata": {},
   "source": [
    "List all the runing instances of the given process_id. *workflow_id* is the id of the process we deployed earlier in the test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "00189a44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"jobs\":[\"645a588a19eb877e986b73d4\"]}'\n"
     ]
    }
   ],
   "source": [
    "# List the executions of the s1tiling workflow \n",
    "exec_list = requests.get(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "print(exec_list.content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "abf7d0be",
   "metadata": {},
   "source": [
    "Get job list and make sure our launched process id, *job_id*, exist in the list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a3257ac5",
   "metadata": {},
   "outputs": [],
   "source": [
    "jobs_list = json.loads(exec_list.content).get(\"jobs\")\n",
    "assert job_id in jobs_list"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "d472931d",
   "metadata": {},
   "source": [
    "In the next step we will ask for our process's status and print its status when it stops running."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "583f856a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "please wait...\n",
      "b'{\"jobId\":\"645a588a19eb877e986b73d4\",\"status\":\"FAILED\",\"message\":\"https://argo.dev.esa-maap.org/workflows/argo/exec-wf-s1tiling-demo-2023-05-09-14-28-26-wpst\",\"progress\":null}'\n"
     ]
    }
   ],
   "source": [
    "# Get the status of the job with the given ID\n",
    "print('please wait...')\n",
    "job_status=None\n",
    "while not job_status or json.loads(job_status.content).get('status') == 'RUNNING':\n",
    "    time.sleep(10)\n",
    "    job_status = requests.get(copa_backend_url+'wpst/processes/{}/jobs/{}'.format(workflow_id, job_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "print(job_status.content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6f959c6c",
   "metadata": {},
   "source": [
    "Remove all processes still in TestJobID.csv."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bef94261",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "645a588a19eb877e986b73d4\n"
     ]
    }
   ],
   "source": [
    "df_to_delete = pd.read_csv(JobIdWD)\n",
    "for x, item in enumerate(df_to_delete.values):\n",
    "    deleted_proc = requests.delete('{}wpst/processes/{}/jobs/{}'.format(copa_backend_url, workflow_id, item[0]), \n",
    "                               headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "    if json.loads(deleted_proc.content).get('id') == item[0]:\n",
    "        print(x)\n",
    "        print(item[0])\n",
    "        df_to_delete.drop(x, axis=0, inplace=True)\n",
    "df_to_delete.to_csv(JobIdWD, index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "b5d4d432",
   "metadata": {},
   "source": [
    "If the workflow did not exist before we start the test in this final step we undeploy it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2a2c29ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b'{\"id\":\"645a588a19eb877e986b73d2\"}'\n"
     ]
    }
   ],
   "source": [
    "#undeploy wf-s1-tiling if found\n",
    "if not process_already_deployed :\n",
    "    delete_response = requests.delete(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
    "    print(delete_response.content)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "311da6fa",
   "metadata": {},
   "source": [
    "End of the test."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Maap",
   "language": "python",
   "name": "maap"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}