diff --git a/wpst_notebook/wpst_echo_test.ipynb b/wpst_notebook/wpst_echo_test.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..58774f2b6946d5171f7dc16a0fb5cb6ef5039dc7
--- /dev/null
+++ b/wpst_notebook/wpst_echo_test.ipynb
@@ -0,0 +1,531 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "8b30c14c",
+ "metadata": {},
+ "source": [
+ "# Test note book \n",
+ "\n",
+ "In this test we're using echo algorythm for it's ease of use and to demonstrate the capabilities and functionality of copa backend API. \n",
+ "\n",
+ "Please make sure you have *copa_backend_url* , *client_id* and *url_token* set in /projects/.maap/maap.ini\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "dd440e2a",
+ "metadata": {},
+ "source": [
+ "### Import and configuration\n",
+ "Let's begin with importing the libraries we need and all the configuration we have, you can find them in /projects/.maap/ folder."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "47236f15",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import configparser\n",
+ "import os\n",
+ "import requests\n",
+ "import json\n",
+ "import time"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "544e720f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Load the file that contains auth values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/auth.ini')\n",
+ "\n",
+ "#Retrieve auth values\n",
+ "email = config['auth']['email']\n",
+ "password = config['auth']['password']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "c25e880c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/projects/demo-scripts/wps_notebook/wpst_test/TestJobID.csv\n"
+ ]
+ }
+ ],
+ "source": [
+ "#Load the file that contains maap values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/maap.ini')\n",
+ "\n",
+ "#Retrieve maap values\n",
+ "copa_backend_url = config['maap']['copa_backend_url']\n",
+ "CLIENT_ID = config['maap']['client_id']\n",
+ "url_token = config['maap']['url_token']\n",
+ "echo_cwl = 'https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/echo-test/workflow.cwl'\n",
+ "\n",
+ "#Location for tmp file for created process\n",
+ "JobIdWD = '{}/TestJobID.csv'.format(os.path.abspath(os.getcwd()))\n",
+ "print(JobIdWD)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "144f3655",
+ "metadata": {},
+ "source": [
+ "### Authentication\n",
+ "Get an authentication token that will allow us to acces copa API service."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "618ebabb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"profile\"})\n",
+ "data = json.loads(response.text)\n",
+ "oauth_token = data['access_token']"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "ba2317cd",
+ "metadata": {},
+ "source": [
+ "### Deployement\n",
+ "Lets start by getting all deployed processes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "944490ee",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"processes\":[]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get deployed cwl workflow\n",
+ "process = requests.get(copa_backend_url+'wpst/processes', headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(process.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "a9a512d7",
+ "metadata": {},
+ "source": [
+ "In the next step we will check if echo process is already deployed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "30893bb2",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get echo-wf id if exist\n",
+ "process_already_deployed = None\n",
+ "if process.content:\n",
+ " for item in json.loads(process.content).get(\"processes\"):\n",
+ " if item.get('title')== 'wf-echo-process':\n",
+ " process_already_deployed = item.get('id')\n",
+ "print(process_already_deployed)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "7af8c2c5",
+ "metadata": {},
+ "source": [
+ "We cannot have multiple deployement of the same process, and in order to test deploying using the deploy endpoint we have to undeploy the existing one first. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "afe6d7d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if process_already_deployed:\n",
+ " delete_response = requests.delete(copa_backend_url+'wpst/processes/{}'.format(process_already_deployed), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ " print(delete_response.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "1edf9f2d",
+ "metadata": {},
+ "source": [
+ "Test deploy endpoint. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "0ab04609",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"processSummary\":{\"id\":\"6455224a9f0e9b5a11d2fd82\",\"title\":\"wf-echo-test\",\"keywords\":null,\"owsContext\":null,\"metadata\":null,\"additionalParameters\":null,\"links\":null,\"version\":null,\"jobControlOptions\":null,\"outputTransmission\":null,\"processDescriptionURL\":\"https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/echo-test/workflow.cwl\",\"abstract\":\"Applies s expressions to EO acquisitions\"}}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "#deploy wf-s1-tiling\n",
+ "\n",
+ "wfstorage = {\"executionUnit\": [{\"href\": \"{}\".format(echo_cwl)}]}\n",
+ "workflow = requests.post(\n",
+ " copa_backend_url+'wpst/processes/', \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token}, \n",
+ " json=wfstorage)\n",
+ "print(workflow.content)\n",
+ "workflow_id = json.loads(workflow.content).get('processSummary').get('id')\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "5f8751b4",
+ "metadata": {},
+ "source": [
+ "### workflow description\n",
+ "\n",
+ "In the next step we will ask the Api endpoint to give us description of the process we just deployed. we can see many informations and most importantly the inputs to launch a job."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "6b42c492",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wf_description = requests.get(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token}) \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "00385eb4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'id': 'input_reference', 'title': 'Input product reference', 'keywords': ['default: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string[]', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Input product reference'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(json.loads(wf_description.content).get('process').get('inputs')[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "2e7f9e7b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "id : input_reference \t\t||\t title : Input product reference\n",
+ "id : s_expression \t\t||\t title : s expression\n",
+ "id : cbn \t\t||\t title : cbn\n",
+ "id : bucket_name \t\t||\t title : Stage-in source bucket\n",
+ "id : copy_dir_or_file \t\t||\t title : Stage-in mode (dir or file)\n",
+ "id : s3_destination \t\t||\t title : Stage-out target folder\n"
+ ]
+ }
+ ],
+ "source": [
+ "for item in json.loads(wf_description.content).get('process').get('inputs'):\n",
+ " print(\"id : {} \\t\\t||\\t title : {}\".format(item.get('id'), item.get('title')))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "ec731150",
+ "metadata": {},
+ "source": [
+ "Based on the Description we have above, we can create a payload of input values. this payload will be used to launch a job."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "7f5047e9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "payload = {\n",
+ " \"inputs\": [\n",
+ " {\n",
+ " \"id\": \"input_file\",\n",
+ " \"data\": \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_message\",\n",
+ " \"data\": \"Hello\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"bucket_name\",\n",
+ " \"data\": \"maap-scientific-data\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"copy_dir_or_file\",\n",
+ " \"data\": \"file\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s3_destination\",\n",
+ " \"data\": \"maap-scientific-data/shared/demo/echo-process-out\",\n",
+ " \"href\": \"\"\n",
+ " }\n",
+ " ],\n",
+ " \"outputs\": [],\n",
+ " \"mode\": \"ASYNC\",\n",
+ " \"response\": \"RAW\"\n",
+ "}"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "b0097062",
+ "metadata": {},
+ "source": [
+ "### Launching a Job\n",
+ "\n",
+ "To not cause confusion, we need to define a job in our context. when launching a process, it create an execution instance, and we will call it a job from now on.\n",
+ "\n",
+ "The folowing request demonstrate the possibility of launching a process and get the job id."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "97f08280",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<Response [201]>\n"
+ ]
+ }
+ ],
+ "source": [
+ "wf_exec = requests.post(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), json=payload, headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "job_id = json.loads(wf_exec.content).get('jobId')\n",
+ "print(wf_exec)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "f8f6995a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"jobs\":[\"6455224b9f0e9b5a11d2fd84\"]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# List the executions of the echo workflow \n",
+ "exec_list = requests.get(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(exec_list.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "a919ee9a",
+ "metadata": {},
+ "source": [
+ "In the previous step we asked the API to provide us with a List of the executions of the CWL workflow with the given ID.\n",
+ "\n",
+ "In the next step we check if the job_id we just created is returned in the list."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "90a62514",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "jobs_list = json.loads(exec_list.content).get(\"jobs\")\n",
+ "assert job_id in jobs_list"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "862e1ee0",
+ "metadata": {},
+ "source": [
+ "#### Job Status\n",
+ "\n",
+ "Get job status."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "addaa77e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "please wait...\n",
+ "b'{\"jobId\":\"6455224b9f0e9b5a11d2fd84\",\"status\":\"FAILED\",\"message\":\"https://argo.dev.esa-maap.org/workflows/argo/exec-wf-echo-test-2023-05-05-15-35-39-wpst\",\"progress\":null}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "print('please wait...')\n",
+ "job_status=None\n",
+ "while not job_status or json.loads(job_status.content).get('status') == 'RUNNING':\n",
+ " time.sleep(10)\n",
+ " job_status = requests.get(copa_backend_url+'wpst/processes/{}/jobs/{}'.format(workflow_id, job_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(job_status.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "d3b3e0ed",
+ "metadata": {},
+ "source": [
+ "#### Delete Job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "21f4db62",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"id\":\"6455224b9f0e9b5a11d2fd84\"}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "deleted_proc = requests.delete('{}wpst/processes/{}/jobs/{}'.format(copa_backend_url, workflow_id, job_id), \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(deleted_proc.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "decb24af",
+ "metadata": {},
+ "source": [
+ "#### Undeploy process."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "3ded80d5",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<Response [200]>"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "requests.delete(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Maap",
+ "language": "python",
+ "name": "maap"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/wpst_notebook/wpst_s1tiling_test.ipynb b/wpst_notebook/wpst_s1tiling_test.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..610e8888bf12ec560e811756e55703c93dafc3ce
--- /dev/null
+++ b/wpst_notebook/wpst_s1tiling_test.ipynb
@@ -0,0 +1,647 @@
+{
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "7e988217",
+ "metadata": {},
+ "source": [
+ "## This notebook serves as a test for wpst endpoints."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "0e9b317f",
+ "metadata": {},
+ "source": [
+ "In order to run this Notebook you need to make sure you have *copa_backend_url* , *s1tiling_cwl*, *client_id* and *url_token* set in /projects/.maap/maap.ini\n",
+ "\n",
+ "**copa_backend_url** : Url for copa bakend, use Url for the corresponding environement.\n",
+ "\n",
+ "**s1tiling_cwl** : Url of s3 location for the cwl of s1tiling algorithm. s1tiling_cwl = https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/s1-tiling-demo/workflow.cwl\n",
+ "\n",
+ "**client_id** : Client ID\n",
+ "\n",
+ "**url_token** : open id auth link\n",
+ "\n",
+ "If the file doesn't exist, please create it using terminal and your favorite editor."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "9986d396",
+ "metadata": {},
+ "source": [
+ "#### Import and conf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "43d3b75a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "import json\n",
+ "import sys\n",
+ "import os\n",
+ "import xml.etree.ElementTree as ET\n",
+ "import time\n",
+ "import sys\n",
+ "import configparser\n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "0c68b487",
+ "metadata": {},
+ "source": [
+ "We load auth configuration from file /projects/.maap/auth.ini"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "fd09ce3a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Load the file that contains auth values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/auth.ini')\n",
+ "\n",
+ "#Retrieve auth values\n",
+ "email = config['auth']['email']\n",
+ "password = config['auth']['password']"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "b4518fef",
+ "metadata": {},
+ "source": [
+ "Load configuration from /projects/.maap/auth.ini"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "bf1b52ba",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/projects/demo-scripts/wps_notebook/wpst_test/TestJobID.csv\n"
+ ]
+ }
+ ],
+ "source": [
+ "#Load the file that contains maap values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/maap.ini')\n",
+ "\n",
+ "#Retrieve maap values\n",
+ "copa_backend_url = config['maap']['copa_backend_url']\n",
+ "CLIENT_ID = config['maap']['client_id']\n",
+ "url_token = config['maap']['url_token']\n",
+ "\n",
+ "#Location for tmp file for created process\n",
+ "JobIdWD = '{}/TestJobID.csv'.format(os.path.abspath(os.getcwd()))\n",
+ "print(JobIdWD)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "b4a0542d",
+ "metadata": {},
+ "source": [
+ "Get access_token using client_id, email and password from configuration"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "ad8e5d4e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'access_token': 'eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICI3dThmWGFxTnFGZHBCSzhLbDRTT1RDeFZodjE3ZUM5dnNZYnFDX2FXZXhZIn0.eyJleHAiOjE2ODM2NzEzMjcsImlhdCI6MTY4MzY0MjUyNywianRpIjoiZTg0MDE1MzUtMDcwZC00MzhjLTg3YmItNDFhM2QyNmY5MWFlIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLmRldi5lc2EtbWFhcC5vcmcvcmVhbG1zL21hYXAiLCJzdWIiOiI3ZmYwNGNlMy02NDRlLTQyOTctYTg2NC1lZWM3MWUyYmI0NWUiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJtYWFwLXRva2VuIiwic2Vzc2lvbl9zdGF0ZSI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCIsImFjciI6IjEiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsib2ZmbGluZV9hY2Nlc3MiLCJDYW1wYWluIEVkaXRvciIsInVtYV9hdXRob3JpemF0aW9uIl19LCJzY29wZSI6InByb2ZpbGUgZW1haWwiLCJzaWQiOiI5ODViMjY2Zi02YTgyLTQzNDUtOTBhNS1kN2M3ZjAzZGNiNjQiLCJlbWFpbF92ZXJpZmllZCI6ZmFsc2UsIm5hbWUiOiJUZXN0IFRlc3QiLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJ0ZXN0QHRlc3QudGVzdCIsImdpdmVuX25hbWUiOiJUZXN0IiwiZmFtaWx5X25hbWUiOiJUZXN0IiwiZW1haWwiOiJ0ZXN0QHRlc3QudGVzdCJ9.k0jMVH33fIQ05_9ok5-SbrZYUE-NHXEZvrtLKuJBF3BWWYzUzUfspKZ4CkA7s5R_4Lbpi1YiNdEuBIzSqR4gShmTA_HnG7sgJVe4R2hxUSH6197xr_GCY8hp4GPMaI4xw0NjPJ-gcXM9O8IJT6j8I0zhsAz_fg8mFO9M7lVv32Oa3QvVqNnspVOl8UA4zl2Hef9rUKDh1Jt--5bBoyxNJzS7Lwz4kslr8gaekFJDLoGho5w9lZVXENo4c-Vxovjmv2uvvUFP1vGmg9kNEBPk-fRc3a10qkBJf0lgCUKxg2v27qlqg_JXnCpdCKsmUbqv8slRCNufhtlZ8Gbp3KMRNQ', 'expires_in': 28800, 'refresh_expires_in': 36000, 'refresh_token': 'eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJjYTI3YWEzYS0zNTM5LTRiMzktODNhNS1kYWFmZDkzZTdmM2QifQ.eyJleHAiOjE2ODM2Nzg1MjcsImlhdCI6MTY4MzY0MjUyNywianRpIjoiZDA2NGQ3MjMtNjgwZC00NzYyLTg3ZTQtOTgzNmRjYWUxODUyIiwiaXNzIjoiaHR0cHM6Ly9hdXRoLmRldi5lc2EtbWFhcC5vcmcvcmVhbG1zL21hYXAiLCJhdWQiOiJodHRwczovL2F1dGguZGV2LmVzYS1tYWFwLm9yZy9yZWFsbXMvbWFhcCIsInN1YiI6IjdmZjA0Y2UzLTY0NGUtNDI5Ny1hODY0LWVlYzcxZTJiYjQ1ZSIsInR5cCI6IlJlZnJlc2giLCJhenAiOiJtYWFwLXRva2VuIiwic2Vzc2lvbl9zdGF0ZSI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCIsInNjb3BlIjoicHJvZmlsZSBlbWFpbCIsInNpZCI6Ijk4NWIyNjZmLTZhODItNDM0NS05MGE1LWQ3YzdmMDNkY2I2NCJ9.Wc3pXRAud6U6xAklFUKztJCwtfz0QVBPx1zljLQ9eT8', 'token_type': 'Bearer', 'not-before-policy': 0, 'session_state': '985b266f-6a82-4345-90a5-d7c7f03dcb64', 'scope': 'profile email'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"profile\"})\n",
+ "data = json.loads(response.text)\n",
+ "print(data)\n",
+ "oauth_token = data['access_token']"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "33c2e6dc",
+ "metadata": {},
+ "source": [
+ "#### WPST process deployement"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "120ba445",
+ "metadata": {},
+ "source": [
+ "For this test we don't want to interfere with existant processes we already have. thus we will list the prcesses we have and check if s1tiling is deployed and get it's id, if not we deploy it.\n",
+ "\n",
+ "WPST end point give possibility to list all the processes deployed"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "d9e3ce0a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"processes\":[]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get deployed cwl workflow\n",
+ "process = requests.get(copa_backend_url+'wpst/processes', headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(process.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "67ec578e",
+ "metadata": {},
+ "source": [
+ "In the next step we will make a loop on the previous response *process* payload in order to get a deployement id for 'wf-s1tiling-demo'. if found we save the id in *process_id* variable"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "e79d0c10",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get wf-s1tiling-demo id if exist\n",
+ "process_already_deployed = None\n",
+ "for item in json.loads(process.content).get(\"processes\"):\n",
+ " if item.get('title')== 'wf-s1tiling-demo':\n",
+ " process_already_deployed = item.get('id')\n",
+ "print(process_already_deployed)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "3c1c1f81",
+ "metadata": {},
+ "source": [
+ "In the next step, if s1tiling is not already deployed, we read s1tiling cwl storage link from configuration and we call the deploy process endpoint using a post method to create a new s1-tiling deployement.\n",
+ "\n",
+ "At the end of this step we will have workflow_id containing our s1tiling id."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "dd258b84",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "200\n",
+ "b'{\"processSummary\":{\"id\":\"645a588a19eb877e986b73d2\",\"title\":\"wf-s1tiling-demo\",\"keywords\":null,\"owsContext\":null,\"metadata\":null,\"additionalParameters\":null,\"links\":null,\"version\":null,\"jobControlOptions\":null,\"outputTransmission\":null,\"processDescriptionURL\":\"https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/s1-tiling-demo/workflow.cwl\",\"abstract\":\"Launch S1-Tiling Algorithm\"}}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "#deploy wf-s1-tiling\n",
+ "url_s1tiling_cwl = config['maap']['s1tiling_cwl']\n",
+ "wfstorage = {\"executionUnit\": [{\"href\": \"{}\".format(url_s1tiling_cwl)}]}\n",
+ "if not process_already_deployed:\n",
+ " workflow = requests.post(\n",
+ " copa_backend_url+'wpst/processes/', \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token}, \n",
+ " json=wfstorage)\n",
+ " print(workflow.status_code)\n",
+ " print(workflow.content)\n",
+ " workflow_id = json.loads(workflow.content).get('processSummary').get('id')\n",
+ "else:\n",
+ " workflow_id = process_already_deployed"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "7fa1b556",
+ "metadata": {},
+ "source": [
+ "#### Launch o process and get status\n",
+ "This mark the succes of deploy test, now we move on to testing launching a process and getting its status."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "bf7952e9",
+ "metadata": {},
+ "source": [
+ "We can get a description of a given process, therefore we can check the inputs needed to luanch this process and other details."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "c7f7ec9a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[{'id': 'input_s1_l1_grd', 'title': 'S1 GRD products folder', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string[]', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Folder containing input S1 L1 GRD product'}, {'id': 's_parallel_proc', 'title': 'Number of parallel process (eg S1 image)', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Number of parallel process (eg S1 image)'}, {'id': 's_ram_per_proc', 'title': 'RAM per process (Mb)', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'RAM Allowed per process in MB'}, {'id': 's_otb_thread', 'title': 'Number of thread in OTB', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Number of thread in otb'}, {'id': 's_light_mode', 'title': 'Run in light mode', 'keywords': ['default: null'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Run in light mode (True/False)'}, {'id': 'bucket_name', 'title': 'Stage-in source bucket', 'keywords': ['default: maap-scientific-data'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Name of the S3 bucket containing the input data'}, {'id': 'copy_dir_or_file', 'title': 'Stage-in mode (dir or file)', 'keywords': ['default: dir'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Flag indicating whether input data is a directory (value=dir) or a file or list of files (value=file)'}, {'id': 's3_destination', 'title': 'Stage-out target folder', 'keywords': ['default: maap-scientific-data/shared/TOBE_COMPLETED'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Bucket name + destination folder in which to transfer result data'}]\n"
+ ]
+ }
+ ],
+ "source": [
+ "wf_description = requests.get(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token}) \n",
+ "print(json.loads(wf_description.content).get('process').get('inputs'))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "f1ad2196",
+ "metadata": {},
+ "source": [
+ "Test that the description of the inputs provided from the backend match the awaited list, if any modification accured on the inputs for s1tiling algorithm it's imperative to add the right values in the list below. Therefore this test make sure we don't have any unwanted regression ."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "de604442",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "8\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Launch s1 tiling wf\n",
+ "awaited_imputs = [ \n",
+ " 'Number of parallel process (eg S1 image)',\n",
+ " 'RAM per process (Mb)',\n",
+ " 'Number of thread in OTB',\n",
+ " 'Run in light mode',\n",
+ " 'Stage-in source bucket',\n",
+ " 'Stage-in mode (dir or file)',\n",
+ " 'Stage-out target folder',\n",
+ " 'S1 GRD products folder',\n",
+ "]\n",
+ "print(len(awaited_imputs))\n",
+ "for item in json.loads(wf_description.content).get('process').get('inputs'):\n",
+ " assert item.get('title') in awaited_imputs"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "1ffaf067",
+ "metadata": {},
+ "source": [
+ "Creating payload for our request to launch an excution of the s1tiling process, the main focus on this part is to load values that the algorithm takes as input.\n",
+ "\n",
+ "Note that we give data correlated to the input id and not input title. the inputs titles above correspond respectively to the ids below ."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "3cdfc486",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "payload = {\n",
+ " \"inputs\": [\n",
+ " {\n",
+ " \"id\": \"s_parallel_proc\",\n",
+ " \"data\": \"1\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_ram_per_proc\",\n",
+ " \"data\": \"8096\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_otb_thread\",\n",
+ " \"data\": \"1\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_light_mode\",\n",
+ " \"data\": \"True\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"bucket_name\",\n",
+ " \"data\": \"maap-scientific-data\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"copy_dir_or_file\",\n",
+ " \"data\": \"dir\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s3_destination\",\n",
+ " \"data\": \"maap-scientific-data/adu/test\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"input_s1_l1_grd\",\n",
+ " \"data\": \"shared/adu/S1_GRD/S2tiles/35SPB\",\n",
+ " \"href\": \"\"\n",
+ " }\n",
+ " ],\n",
+ " \"outputs\": [],\n",
+ " \"mode\": \"ASYNC\",\n",
+ " \"response\": \"RAW\"\n",
+ "}"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "ccada0dc",
+ "metadata": {},
+ "source": [
+ "Run the process using the above payload."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "f683815d",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<Response [201]>\n"
+ ]
+ }
+ ],
+ "source": [
+ "wf_exec = requests.post(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), json=payload, headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "job_id = json.loads(wf_exec.content).get('jobId')\n",
+ "print(wf_exec)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "567b516c",
+ "metadata": {},
+ "source": [
+ "Write jobId in TestJobID.csv to save the jobs launched in this test, to be removed all at the end. Cn case of a falure of tests after this step, the id will always be in TestJobID.csv and when we reach the step 16 we can remove them all at once."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "c7177b32",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "jobIds = []\n",
+ "if os.path.exists(JobIdWD):\n",
+ " df2 = pd.read_csv(JobIdWD)\n",
+ " for item in df2.values:\n",
+ " jobIds.append(str(item[0]))\n",
+ "\n",
+ "if job_id not in jobIds:\n",
+ " jobIds.append(job_id)\n",
+ "df = pd.DataFrame({'jobID':jobIds})\n",
+ "df.to_csv(JobIdWD, index=False)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "03c909c6",
+ "metadata": {},
+ "source": [
+ "List all the runing instances of the given process_id. *workflow_id* is the id of the process we deployed earlier in the test"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "00189a44",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"jobs\":[\"645a588a19eb877e986b73d4\"]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# List the executions of the s1tiling workflow \n",
+ "exec_list = requests.get(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(exec_list.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "abf7d0be",
+ "metadata": {},
+ "source": [
+ "Get job list and make sure our launched process id, *job_id*, exist in the list."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "a3257ac5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "jobs_list = json.loads(exec_list.content).get(\"jobs\")\n",
+ "assert job_id in jobs_list"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "d472931d",
+ "metadata": {},
+ "source": [
+ "In the next step we will ask for our process's status and print its status when it stops running."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "583f856a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "please wait...\n",
+ "b'{\"jobId\":\"645a588a19eb877e986b73d4\",\"status\":\"FAILED\",\"message\":\"https://argo.dev.esa-maap.org/workflows/argo/exec-wf-s1tiling-demo-2023-05-09-14-28-26-wpst\",\"progress\":null}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Get the status of the job with the given ID\n",
+ "print('please wait...')\n",
+ "job_status=None\n",
+ "while not job_status or json.loads(job_status.content).get('status') == 'RUNNING':\n",
+ " time.sleep(10)\n",
+ " job_status = requests.get(copa_backend_url+'wpst/processes/{}/jobs/{}'.format(workflow_id, job_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(job_status.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "6f959c6c",
+ "metadata": {},
+ "source": [
+ "Remove all processes still in TestJobID.csv."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "bef94261",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0\n",
+ "645a588a19eb877e986b73d4\n"
+ ]
+ }
+ ],
+ "source": [
+ "df_to_delete = pd.read_csv(JobIdWD)\n",
+ "for x, item in enumerate(df_to_delete.values):\n",
+ " deleted_proc = requests.delete('{}wpst/processes/{}/jobs/{}'.format(copa_backend_url, workflow_id, item[0]), \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ " if json.loads(deleted_proc.content).get('id') == item[0]:\n",
+ " print(x)\n",
+ " print(item[0])\n",
+ " df_to_delete.drop(x, axis=0, inplace=True)\n",
+ "df_to_delete.to_csv(JobIdWD, index=False)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "b5d4d432",
+ "metadata": {},
+ "source": [
+ "If the workflow did not exist before we start the test in this final step we undeploy it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "id": "2a2c29ef",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"id\":\"645a588a19eb877e986b73d2\"}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "#undeploy wf-s1-tiling if found\n",
+ "if not process_already_deployed :\n",
+ " delete_response = requests.delete(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ " print(delete_response.content)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "id": "311da6fa",
+ "metadata": {},
+ "source": [
+ "End of the test."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Maap",
+ "language": "python",
+ "name": "maap"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}