From a57ded38bf4c7f67813ee36c57f16aa65bdd22c4 Mon Sep 17 00:00:00 2001
From: Aymane HAFIANE <aymane.hafiane@capgemini.com>
Date: Wed, 10 May 2023 10:07:42 +0200
Subject: [PATCH] BIOMASS-2980 notebook test echo wf
---
wpst_notebook_test/wpst_echo_test.ipynb | 529 ++++++++++++++++++++++++
1 file changed, 529 insertions(+)
create mode 100644 wpst_notebook_test/wpst_echo_test.ipynb
diff --git a/wpst_notebook_test/wpst_echo_test.ipynb b/wpst_notebook_test/wpst_echo_test.ipynb
new file mode 100644
index 0000000..86877eb
--- /dev/null
+++ b/wpst_notebook_test/wpst_echo_test.ipynb
@@ -0,0 +1,529 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "8b30c14c",
+ "metadata": {},
+ "source": [
+ "# Test note book \n",
+ "\n",
+ "In this test we're using echo algorythm for it's ease of use and to demonstrate the capabilities and functionality of copa backend API."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dd440e2a",
+ "metadata": {},
+ "source": [
+ "### Import and configuration\n",
+ "Let's begin with importing the libraries we need and all the configuration we have, you can find them in /projects/.maap/ folder."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "47236f15",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "import json\n",
+ "import sys\n",
+ "import os\n",
+ "import xml.etree.ElementTree as ET\n",
+ "import time\n",
+ "import sys\n",
+ "import configparser\n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "544e720f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Load the file that contains auth values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/auth.ini')\n",
+ "\n",
+ "#Retrieve auth values\n",
+ "email = config['auth']['email']\n",
+ "password = config['auth']['password']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "c25e880c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/projects/demo-scripts/wps_notebook/wpst_test/TestJobID.csv\n"
+ ]
+ }
+ ],
+ "source": [
+ "#Load the file that contains maap values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/maap.ini')\n",
+ "\n",
+ "#Retrieve maap values\n",
+ "copa_backend_url = config['maap']['copa_backend_url']\n",
+ "CLIENT_ID = config['maap']['client_id']\n",
+ "url_token = config['maap']['url_token']\n",
+ "echo_cwl = config['maap']['echo_cwl']\n",
+ "\n",
+ "#Location for tmp file for created process\n",
+ "JobIdWD = '{}/TestJobID.csv'.format(os.path.abspath(os.getcwd()))\n",
+ "print(JobIdWD)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "144f3655",
+ "metadata": {},
+ "source": [
+ "### Authentication\n",
+ "Get an authentication token that will allow us to acces copa API service."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "618ebabb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"profile\"})\n",
+ "data = json.loads(response.text)\n",
+ "oauth_token = data['access_token']"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ba2317cd",
+ "metadata": {},
+ "source": [
+ "### Deployement\n",
+ "Lets start by getting all deployed processes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "944490ee",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"processes\":[]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get deployed cwl workflow\n",
+ "process = requests.get(copa_backend_url+'wpst/processes', headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(process.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a9a512d7",
+ "metadata": {},
+ "source": [
+ "In the next step we will check if echo process is already deployed."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "30893bb2",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "# get echo-wf id if exist\n",
+ "process_already_deployed = None\n",
+ "if process.content:\n",
+ " for item in json.loads(process.content).get(\"processes\"):\n",
+ " if item.get('title')== 'wf-echo-test':\n",
+ " process_already_deployed = item.get('id')\n",
+ "print(process_already_deployed)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7af8c2c5",
+ "metadata": {},
+ "source": [
+ "We cannot have multiple deployement of the same process, and in order to test deploying using the deploy endpoint we have to undeploy the existing one first. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "afe6d7d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if process_already_deployed:\n",
+ " delete_response = requests.delete(copa_backend_url+'wpst/processes/{}'.format(process_already_deployed), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ " print(delete_response.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1edf9f2d",
+ "metadata": {},
+ "source": [
+ "Test deploy endpoint. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "0ab04609",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"processSummary\":{\"id\":\"6455224a9f0e9b5a11d2fd82\",\"title\":\"wf-echo-test\",\"keywords\":null,\"owsContext\":null,\"metadata\":null,\"additionalParameters\":null,\"links\":null,\"version\":null,\"jobControlOptions\":null,\"outputTransmission\":null,\"processDescriptionURL\":\"https://s3public.oss.eu-west-0.prod-cloud-ocb.orange-business.com/cwl/echo-test/workflow.cwl\",\"abstract\":\"Applies s expressions to EO acquisitions\"}}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "#deploy wf-s1-tiling\n",
+ "\n",
+ "wfstorage = {\"executionUnit\": [{\"href\": \"{}\".format(echo_cwl)}]}\n",
+ "workflow = requests.post(\n",
+ " copa_backend_url+'wpst/processes/', \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token}, \n",
+ " json=wfstorage)\n",
+ "print(workflow.content)\n",
+ "workflow_id = json.loads(workflow.content).get('processSummary').get('id')\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5f8751b4",
+ "metadata": {},
+ "source": [
+ "### workflow description\n",
+ "\n",
+ "In the next step we will ask the Api endpoint to give us description of the process we just deployed. we can see many informations and most importantly the inputs to launch a job."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "6b42c492",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wf_description = requests.get(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token}) \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "00385eb4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'id': 'input_reference', 'title': 'Input product reference', 'keywords': ['default: https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A'], 'owsContext': None, 'metadata': None, 'additionalParameters': None, 'links': None, 'minOccurs': None, 'maxOccurs': None, 'formats': [{'mimeType': 'string[]', 'schema': None, 'encoding': None, 'maximumMegabytes': None, 'default': False}], 'abstract': 'Input product reference'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(json.loads(wf_description.content).get('process').get('inputs')[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "2e7f9e7b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "id : input_reference \t\t||\t title : Input product reference\n",
+ "id : s_expression \t\t||\t title : s expression\n",
+ "id : cbn \t\t||\t title : cbn\n",
+ "id : bucket_name \t\t||\t title : Stage-in source bucket\n",
+ "id : copy_dir_or_file \t\t||\t title : Stage-in mode (dir or file)\n",
+ "id : s3_destination \t\t||\t title : Stage-out target folder\n"
+ ]
+ }
+ ],
+ "source": [
+ "for item in json.loads(wf_description.content).get('process').get('inputs'):\n",
+ " print(\"id : {} \\t\\t||\\t title : {}\".format(item.get('id'), item.get('title')))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ec731150",
+ "metadata": {},
+ "source": [
+ "Based on the Description we have above, we can create a payload of input values. this payload will be used to launch a job."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "7f5047e9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "payload = {\n",
+ " \"inputs\": [\n",
+ " {\n",
+ " \"id\": \"input_reference\",\n",
+ " \"data\": \"https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items/S2B_36RTT_20191205_0_L2A\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_expression\",\n",
+ " \"data\": \"(/ (- B05 B02) (+ B05 B02))\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"cbn\",\n",
+ " \"data\": \"nbr\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s_light_mode\",\n",
+ " \"data\": \"True\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"bucket_name\",\n",
+ " \"data\": \"maap-scientific-data\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"copy_dir_or_file\",\n",
+ " \"data\": \"dir\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " {\n",
+ " \"id\": \"s3_destination\",\n",
+ " \"data\": \"maap-scientific-data/adu/test\",\n",
+ " \"href\": \"\"\n",
+ " },\n",
+ " ],\n",
+ " \"outputs\": [],\n",
+ " \"mode\": \"ASYNC\",\n",
+ " \"response\": \"RAW\"\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b0097062",
+ "metadata": {},
+ "source": [
+ "### Launching a Job\n",
+ "\n",
+ "To not cause confusion, we need to define a job in our context. when launching a process, it create an execution instance, and we will call it a job from now on.\n",
+ "\n",
+ "The folowing request demonstrate the possibility of launching a process and get the job id."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "id": "97f08280",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "<Response [201]>\n"
+ ]
+ }
+ ],
+ "source": [
+ "wf_exec = requests.post(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), json=payload, headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "job_id = json.loads(wf_exec.content).get('jobId')\n",
+ "print(wf_exec)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "id": "f8f6995a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"jobs\":[\"6455224b9f0e9b5a11d2fd84\"]}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "# List the executions of the echo workflow \n",
+ "exec_list = requests.get(copa_backend_url+'wpst/processes/{}/jobs'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(exec_list.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a919ee9a",
+ "metadata": {},
+ "source": [
+ "In the previous step we asked the API to provide us with a List of the executions of the CWL workflow with the given ID.\n",
+ "\n",
+ "In the next step we check if the job_id we just created is returned in the list."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "90a62514",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "jobs_list = json.loads(exec_list.content).get(\"jobs\")\n",
+ "assert job_id in jobs_list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "862e1ee0",
+ "metadata": {},
+ "source": [
+ "#### Job Status\n",
+ "\n",
+ "Get job status."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "id": "addaa77e",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "please wait...\n",
+ "b'{\"jobId\":\"6455224b9f0e9b5a11d2fd84\",\"status\":\"FAILED\",\"message\":\"https://argo.dev.esa-maap.org/workflows/argo/exec-wf-echo-test-2023-05-05-15-35-39-wpst\",\"progress\":null}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "print('please wait...')\n",
+ "job_status=None\n",
+ "while not job_status or json.loads(job_status.content).get('status') == 'RUNNING':\n",
+ " time.sleep(10)\n",
+ " job_status = requests.get(copa_backend_url+'wpst/processes/{}/jobs/{}'.format(workflow_id, job_id), headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(job_status.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d3b3e0ed",
+ "metadata": {},
+ "source": [
+ "#### Delete Job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "id": "21f4db62",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "b'{\"id\":\"6455224b9f0e9b5a11d2fd84\"}'\n"
+ ]
+ }
+ ],
+ "source": [
+ "deleted_proc = requests.delete('{}wpst/processes/{}/jobs/{}'.format(copa_backend_url, workflow_id, job_id), \n",
+ " headers = {'Authorization': 'Bearer '+ oauth_token})\n",
+ "print(deleted_proc.content)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "decb24af",
+ "metadata": {},
+ "source": [
+ "#### Undeploy process."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "id": "3ded80d5",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<Response [200]>"
+ ]
+ },
+ "execution_count": 18,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "requests.delete(copa_backend_url+'wpst/processes/{}'.format(workflow_id), headers = {'Authorization': 'Bearer '+ oauth_token})"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Maap",
+ "language": "python",
+ "name": "maap"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--
GitLab