Merge branch 'BIOMASS-2980-test-nb' into 'master'

Biomass 2980 test nb

See merge request !14

wpst_notebook/wpst_echo_test.ipynb

+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "8b30c14c",
+   "metadata": {},
+   "source": [
+    "# Test note book \n",
+    "\n",
+    "In this test we're using echo algorythm for it's ease of use and to demonstrate the capabilities and functionality of copa backend API. \n",
+    "\n",
+    "Please make sure you have *copa_backend_url* , *client_id* and *url_token* set in /projects/.maap/maap.ini\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "dd440e2a",
+   "metadata": {},
+   "source": [
+    "### Import and configuration\n",
+    "Let's begin with importing the libraries we need and all the configuration we have, you can find them in /projects/.maap/ folder."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "47236f15",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import configparser\n",
+    "import os\n",
+    "import requests\n",
+    "import json\n",
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "544e720f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#Load the file that contains auth values\n",
+    "config = configparser.ConfigParser()\n",
+    "config.read('/projects/.maap/auth.ini')\n",
+    "\n",
+    "#Retrieve auth values\n",
+    "email = config['auth']['email']\n",
+    "password = config['auth']['password']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "c25e880c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/projects/demo-scripts/wps_notebook/wpst_test/TestJobID.csv\n"
+     ]
+    }
+   ],
+   "source": [
+    "#Load the file that contains maap values\n",
+    "config = configparser.ConfigParser()\n",
+    "config.read('/projects/.maap/maap.ini')\n",
+    "\n",
+    "#Retrieve maap values\n",
+    "copa_backend_url = config['maap']['copa_backend_url']\n",
+    "CLIENT_ID = config['maap']['client_id']\n",
+    "url_token = config['maap']['url_token']\n",
+    "echo_cwl = 'https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/echo-test/workflow.cwl'\n",
+    "\n",
+    "#Location for tmp file for created process\n",
+    "JobIdWD = '{}/TestJobID.csv'.format(os.path.abspath(os.getcwd()))\n",
+    "print(JobIdWD)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "144f3655",
+   "metadata": {},
+   "source": [
+    "### Authentication\n",
+    "Get an authentication token that will allow us to acces copa API service."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "618ebabb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"profile\"})\n",
+    "data = json.loads(response.text)\n",
+    "oauth_token = data['access_token']"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "ba2317cd",
+   "metadata": {},
+   "source": [
+    "### Deployement\n",
+    "Lets start by getting all deployed processes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "944490ee",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b'{\"processes\":[]}'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# get deployed cwl workflow\n",
+    "process = requests.get(copa_backend_url+'wpst/processes', headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "print(process.content)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "a9a512d7",
+   "metadata": {},
+   "source": [
+    "In the next step we will check if echo process is already deployed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "30893bb2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "# get echo-wf id if exist\n",
+    "process_already_deployed = None\n",
+    "if process.content:\n",
+    "    for item in json.loads(process.content).get(\"processes\"):\n",
+    "        if item.get('title')== 'wf-echo-process':\n",
+    "            process_already_deployed = item.get('id')\n",
+    "print(process_already_deployed)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "7af8c2c5",
+   "metadata": {},
+   "source": [
+    "We cannot have multiple deployement of the same process, and in order to test deploying using the deploy endpoint we have to undeploy the existing one first. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "afe6d7d5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if process_already_deployed:\n",
+    "    delete_response = requests.delete(copa_backend_url+'wpst/processes/{}'.format(process_already_deployed), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "    print(delete_response.content)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "1edf9f2d",
+   "metadata": {},
+   "source": [
+    "Test deploy endpoint. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "0ab04609",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b'{\"processSummary\":{\"id\":\"6455224a9f0e9b5a11d2fd82\",\"title\":\"wf-echo-test\",\"keywords\":null,\"owsContext\":null,\"metadata\":null,\"additionalParameters\":null,\"links\":null,\"version\":null,\"jobControlOptions\":null,\"outputTransmission\":null,\"processDescriptionURL\":\"https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/echo-test/workflow.cwl\",\"abstract\":\"Applies s expressions to EO acquisitions\"}}'\n"
+     ]
+    }
+   ],
+   "source": [
+    "#deploy wf-s1-tiling\n",
+    "\n",
+    "wfstorage =  {\"executionUnit\": [{\"href\": \"{}\".format(echo_cwl)}]}\n",
+    "workflow = requests.post(\n",
+    "                        copa_backend_url+'wpst/processes/', \n",
+    "                        headers = {'Authorization': 'Bearer '+ oauth_token}, \n",
+    "                        json=wfstorage)\n",
+    "print(workflow.content)\n",
+    "workflow_id = json.loads(workflow.content).get('processSummary').get('id')\n"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "5f8751b4",
+   "metadata": {},
+   "source": [
+    "### workflow description\n",
+    "\n",
+    "In the next step we will ask the Api endpoint to give us description of the process we just deployed. we can see many informations and most importantly the inputs to launch a job."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "6b42c492",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "wf_description = requests.get(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token}) \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "00385eb4",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'id': 'input_reference', 'title': 'Input product reference', 'keywords': ['default: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string[]', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Input product reference'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(json.loads(wf_description.content).get('process').get('inputs')[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "2e7f9e7b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "id : input_reference \t\t||\t title : Input product reference\n",
+      "id : s_expression \t\t||\t title : s expression\n",
+      "id : cbn \t\t||\t title : cbn\n",
+      "id : bucket_name \t\t||\t title : Stage-in source bucket\n",
+      "id : copy_dir_or_file \t\t||\t title : Stage-in mode (dir or file)\n",
+      "id : s3_destination \t\t||\t title : Stage-out target folder\n"
+     ]
+    }
+   ],
+   "source": [
+    "for item in json.loads(wf_description.content).get('process').get('inputs'):\n",
+    "    print(\"id : {} \\t\\t||\\t title : {}\".format(item.get('id'), item.get('title')))"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "ec731150",
+   "metadata": {},
+   "source": [
+    "Based on the Description we have above, we can create a payload of input values. this payload will be used to launch a job."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "7f5047e9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "payload = {\n",
+    "  \"inputs\": [\n",
+    "    {\n",
+    "      \"id\": \"input_file\",\n",
+    "      \"data\": \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A\",\n",
+    "      \"href\": \"\"\n",
+    "    },\n",
+    "    {\n",
+    "      \"id\": \"s_message\",\n",
+    "      \"data\": \"Hello\",\n",
+    "      \"href\": \"\"\n",
+    "    },\n",
+    "    {\n",
+    "      \"id\": \"bucket_name\",\n",
+    "      \"data\": \"maap-scientific-data\",\n",
+    "      \"href\": \"\"\n",
+    "    },\n",
+    "    {\n",
+    "      \"id\": \"copy_dir_or_file\",\n",
+    "      \"data\": \"file\",\n",
+    "      \"href\": \"\"\n",
+    "    },\n",
+    "    {\n",
+    "      \"id\": \"s3_destination\",\n",
+    "      \"data\": \"maap-scientific-data/shared/demo/echo-process-out\",\n",
+    "      \"href\": \"\"\n",
+    "    }\n",
+    "  ],\n",
+    "  \"outputs\": [],\n",
+    "  \"mode\": \"ASYNC\",\n",
+    "  \"response\": \"RAW\"\n",
+    "}"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "b0097062",
+   "metadata": {},
+   "source": [
+    "### Launching a Job\n",
+    "\n",
+    "To not cause confusion, we need to define a job in our context. when launching a process, it create an execution instance, and we will call it a job from now on.\n",
+    "\n",
+    "The folowing request demonstrate the possibility of launching a process and get the job id."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "97f08280",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<Response [201]>\n"
+     ]
+    }
+   ],
+   "source": [
+    "wf_exec = requests.post(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), json=payload, headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "job_id = json.loads(wf_exec.content).get('jobId')\n",
+    "print(wf_exec)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "f8f6995a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b'{\"jobs\":[\"6455224b9f0e9b5a11d2fd84\"]}'\n"
+     ]
+    }
+   ],
+   "source": [
+    "# List the executions of the echo workflow \n",
+    "exec_list = requests.get(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "print(exec_list.content)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "a919ee9a",
+   "metadata": {},
+   "source": [
+    "In the previous step we asked the API to provide us with a List of the executions of the CWL workflow with the given ID.\n",
+    "\n",
+    "In the next step we check if the job_id we just created is returned in the list."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "90a62514",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "jobs_list = json.loads(exec_list.content).get(\"jobs\")\n",
+    "assert job_id in jobs_list"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "862e1ee0",
+   "metadata": {},
+   "source": [
+    "#### Job Status\n",
+    "\n",
+    "Get job status."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "addaa77e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "please wait...\n",
+      "b'{\"jobId\":\"6455224b9f0e9b5a11d2fd84\",\"status\":\"FAILED\",\"message\":\"https://argo.dev.esa-maap.org/workflows/argo/exec-wf-echo-test-2023-05-05-15-35-39-wpst\",\"progress\":null}'\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('please wait...')\n",
+    "job_status=None\n",
+    "while not job_status or json.loads(job_status.content).get('status') == 'RUNNING':\n",
+    "    time.sleep(10)\n",
+    "    job_status = requests.get(copa_backend_url+'wpst/processes/{}/jobs/{}'.format(workflow_id, job_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "print(job_status.content)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "d3b3e0ed",
+   "metadata": {},
+   "source": [
+    "#### Delete Job"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "21f4db62",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "b'{\"id\":\"6455224b9f0e9b5a11d2fd84\"}'\n"
+     ]
+    }
+   ],
+   "source": [
+    "deleted_proc = requests.delete('{}wpst/processes/{}/jobs/{}'.format(copa_backend_url, workflow_id, job_id), \n",
+    "                               headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+    "print(deleted_proc.content)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "decb24af",
+   "metadata": {},
+   "source": [
+    "#### Undeploy process."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "3ded80d5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Response [200]>"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "requests.delete(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Maap",
+   "language": "python",
+   "name": "maap"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}