diff --git a/wps_notebook/Scalability_Demonstration.ipynb b/wps_notebook/Scalability_Demonstration.ipynb
index e384e21a3678af96e26aca2fd241dd59b230824f..bfae1519b04e5d2dc1cd2cfd773625cfe6198997 100644
--- a/wps_notebook/Scalability_Demonstration.ipynb
+++ b/wps_notebook/Scalability_Demonstration.ipynb
@@ -14,7 +14,8 @@
"import time\n",
"import sys\n",
"from wpsToolsBox import wpsTB\n",
- "import configparser"
+ "import configparser\n",
+ "import pandas as pd"
]
},
{
@@ -106,7 +107,7 @@
"dirname = os.getcwd()\n",
"xml_file = os.path.join(dirname, 'xml_file/wps_readstwo.xml')\n",
"\n",
- "jobID = wps.wps_execute_process(xml_file, 6)"
+ "jobID = wps.wps_execute_process(xml_file, 1)"
]
},
{
@@ -120,6 +121,7 @@
" #FIXME retrieve the id of the user\n",
" url = copa_backend_url + 'wps/getStatus?userId=' + user_id\n",
" body = { \"jobID\": Id }\n",
+ " print(Id)\n",
" response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)\n",
" parsed = json.loads(response.text)\n",
"\n",
@@ -147,6 +149,14 @@
"outputs": [],
"source": [
"print(\"--- Delete all jobs---\")\n",
+ "df2 = pd.read_csv(\"jobIDs.csv\")\n",
+ "print('List of the backuped job IDs')\n",
+ "jobs=[]\n",
+ "for i in df2.values :\n",
+ " jobs.append(i[0])\n",
+ "print(jobs)\n",
+ "#jobID =['62221dcb310b59000170059d','62221c61310b590001700599','62221c8d310b59000170059b']\n",
+ "\n",
"for Id in jobID: \n",
" url = copa_backend_url + 'wps/dismiss?userId=' + user_id\n",
" body = { \"jobID\": Id }\n",
diff --git a/wps_notebook/Scalability_aksk_biopal_fh.ipynb b/wps_notebook/Scalability_aksk_biopal_fh.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..a8b690f7dcd70bd3c88188dcc0c1e5d504cbe635
--- /dev/null
+++ b/wps_notebook/Scalability_aksk_biopal_fh.ipynb
@@ -0,0 +1,197 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import requests\n",
+ "import json\n",
+ "import sys\n",
+ "import os\n",
+ "import xml.etree.ElementTree as ET\n",
+ "import time\n",
+ "import sys\n",
+ "from wpsToolsBox import wpsTB\n",
+ "import configparser\n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Load the file that contains auth values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/auth.ini')\n",
+ "\n",
+ "#Retrieve auth values\n",
+ "user_id = config['auth']['user_id']\n",
+ "email = config['auth']['email']\n",
+ "password = config['auth']['password']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Load the file that contains maap values\n",
+ "config = configparser.ConfigParser()\n",
+ "config.read('/projects/.maap/maap.ini')\n",
+ "\n",
+ "#Retrieve maap values\n",
+ "copa_backend_url = config['maap']['copa_backend_url']\n",
+ "CLIENT_ID = config['maap']['client_id']\n",
+ "url_token = config['maap']['url_token']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "response = requests.post(url_token, data={'client_id': CLIENT_ID, 'username': email, 'password': password, \"grant_type\": \"password\", \"scope\": \"openid+profile\"})\n",
+ "data = json.loads(response.text)\n",
+ "print(data)\n",
+ "oauth_token = data['access_token']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wps = wpsTB(oauth_token, copa_backend_url, user_id)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wps.wps_info()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wps.wps_processes()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "wps.wps_process_info('aksk-biopal-fh')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dirname = os.getcwd()\n",
+ "xml_file = os.path.join(dirname, 'xml_file/wps_aksk-biopal-fh.xml')\n",
+ "\n",
+ "jobID = wps.wps_execute_process(xml_file, 100)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"--- GetStatus---\")\n",
+ "for Id in jobID: \n",
+ " #FIXME retrieve the id of the user\n",
+ " url = copa_backend_url + 'wps/getStatus?userId=' + user_id\n",
+ " body = { \"jobID\": Id }\n",
+ " print(Id)\n",
+ " response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)\n",
+ " parsed = json.loads(response.text)\n",
+ "\n",
+ " print(json.dumps(parsed, indent=2, sort_keys=True))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"--- GetResult---\")\n",
+ "for Id in jobID: \n",
+ " #FIXME retrieve the id of the user\n",
+ " url = copa_backend_url + 'wps/'+ Id +'?userId=' + user_id\n",
+ " response = requests.get(url, headers = {'Authorization': 'Bearer '+oauth_token})\n",
+ " print(response.text)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"--- Delete all jobs---\")\n",
+ "\n",
+ "df2 = pd.read_csv(\"jobIDs.csv\")\n",
+ "print('List of the backuped job IDs')\n",
+ "jobs=[]\n",
+ "for i in df2.values :\n",
+ " jobs.append(i[0])\n",
+ "print(jobs)\n",
+ "#jobID =['623c840d4f4c630001aca946', '623c840f4f4c630001aca948', '623c84104f4c630001aca94a', '623c84124f4c630001aca94c', '623c84144f4c630001aca94e', '623c84154f4c630001aca950', '623c84174f4c630001aca952', '623c84184f4c630001aca954', '623c841a4f4c630001aca956', '623c841b4f4c630001aca958', '623c841d4f4c630001aca95a', '623c841e4f4c630001aca95c', '623c84204f4c630001aca95e', '623c84214f4c630001aca960', '623c84234f4c630001aca962', '623c84244f4c630001aca964', '623c84264f4c630001aca966', '623c84274f4c630001aca968', '623c84294f4c630001aca96a', '623c842a4f4c630001aca96c', '623c842c4f4c630001aca96e', '623c842d4f4c630001aca970', '623c842f4f4c630001aca972', '623c84304f4c630001aca974', '623c84324f4c630001aca976', '623c84334f4c630001aca978', '623c84354f4c630001aca97a', '623c84364f4c630001aca97c', '623c84384f4c630001aca97e', '623c84394f4c630001aca980', '623c843b4f4c630001aca982', '623c843c4f4c630001aca984', '623c843e4f4c630001aca986', '623c843f4f4c630001aca988', '623c84414f4c630001aca98a', '623c84424f4c630001aca98c', '623c84444f4c630001aca98e', '623c84454f4c630001aca990', '623c84474f4c630001aca992', '623c84484f4c630001aca994', '623c844a4f4c630001aca996', '623c844b4f4c630001aca998', '623c844d4f4c630001aca99a', '623c844e4f4c630001aca99c', '623c84504f4c630001aca99e', '623c84514f4c630001aca9a0', '623c84534f4c630001aca9a2', '623c84554f4c630001aca9a4', '623c84564f4c630001aca9a6', '623c84584f4c630001aca9a8', '623c84594f4c630001aca9aa', '623c845a4f4c630001aca9ac', '623c845c4f4c630001aca9ae', '623c845d4f4c630001aca9b0', '623c845f4f4c630001aca9b2', '623c84604f4c630001aca9b4', '623c84624f4c630001aca9b6', '623c84634f4c630001aca9b8', '623c84654f4c630001aca9ba', '623c84664f4c630001aca9bc', '623c84684f4c630001aca9be', '623c846a4f4c630001aca9c0', '623c846b4f4c630001aca9c2', '623c846d4f4c630001aca9c4', '623c846e4f4c630001aca9c6', '623c84704f4c630001aca9c8', '623c84714f4c630001aca9ca', '623c84734f4c630001aca9cc', '623c84744f4c630001aca9ce', '623c84764f4c630001aca9d0', '623c84774f4c630001aca9d2', '623c84794f4c630001aca9d4', '623c847a4f4c630001aca9d6', '623c847c4f4c630001aca9d8', '623c847d4f4c630001aca9da', '623c847f4f4c630001aca9dc', '623c84804f4c630001aca9de', '623c84824f4c630001aca9e0', '623c84834f4c630001aca9e2', '623c84854f4c630001aca9e4', '623c84864f4c630001aca9e6', '623c84884f4c630001aca9e8', '623c84894f4c630001aca9ea', '623c848b4f4c630001aca9ec', '623c848c4f4c630001aca9ee', '623c848e4f4c630001aca9f0', '623c848f4f4c630001aca9f2', '623c84914f4c630001aca9f4', '623c84924f4c630001aca9f6', '623c84944f4c630001aca9f8', '623c84954f4c630001aca9fa', '623c84974f4c630001aca9fc', '623c84984f4c630001aca9fe', '623c849a4f4c630001acaa00', '623c849b4f4c630001acaa02', '623c849d4f4c630001acaa04', '623c849e4f4c630001acaa06', '623c84a04f4c630001acaa08', '623c84a14f4c630001acaa0a', '623c84a34f4c630001acaa0c']\n",
+ "for Id in jobID: \n",
+ " url = copa_backend_url + 'wps/dismiss?userId=' + user_id\n",
+ " body = { \"jobID\": Id }\n",
+ " response = requests.post(url, headers = {'Authorization': 'Bearer '+oauth_token}, json = body)\n",
+ " parsed = json.loads(response.text)\n",
+ " print(json.dumps(parsed, indent=2, sort_keys=True))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Maap",
+ "language": "python",
+ "name": "maap"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/wps_notebook/wpsToolsBox.py b/wps_notebook/wpsToolsBox.py
index f789a49c1d777a6d8c0afedc687540c0479127bf..90a6972d4f6f7a3f78f6a4f06d168caaf6efc59e 100644
--- a/wps_notebook/wpsToolsBox.py
+++ b/wps_notebook/wpsToolsBox.py
@@ -5,6 +5,7 @@ sys.path.insert(0, '../common')
from owslib.wps import WebProcessingService
import time
import json
+import pandas as pd
class wpsTB:
@@ -95,6 +96,13 @@ class wpsTB:
time.sleep(1)
n_iter += 1
+ # backup all the job IDs in a csv file to be abble to delete it at the end even if the notebook failed
+ # dictionary of lists
+ dict = {'jobIDs': job}
+ # creating a dataframe from a dictionary
+ df = pd.DataFrame(dict)
+ df.to_csv('jobIDs.csv',header = True, index= False)
+
while len(job) != 0:
for Id in job:
url = self.copa_backend_url + 'wps/getStatus?userId=' + self.user_id
diff --git a/wps_notebook/xml_file/wps_aksk-biopal-fh.xml b/wps_notebook/xml_file/wps_aksk-biopal-fh.xml
new file mode 100644
index 0000000000000000000000000000000000000000..82addc46697c3e7518cb034c9933fb34f6cb75f8
--- /dev/null
+++ b/wps_notebook/xml_file/wps_aksk-biopal-fh.xml
@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?><ExecuteRequestType mode="async" response="document" service="WPS" version="2.0.0" xmlns:xlin="http://www.w3.org/1999/xlink" xmlns:wfs="http://www.opengis.net/wfs/2.1" xmlns:sld="http://www.opengis.net/sld/1.2" xmlns:ows="http://www.opengis.net/ows/2.0" xmlns:kml="http://www.opengis.net/kml/2.2" xmlns:ns8="http://www.opengis.net/se/2.0/core" xmlns:ns13="http://www.opengis.net/gml" xmlns:ns9="http://www.opengis.net/fes/2.1" xmlns:ns12="http://www.opengis.net/ows-context" xmlns:ns11="http://www.opengis.net/se/2.0/raster" xmlns:ns10="http://www.opengis.net/se/2.0/thematic" xmlns:ns16="urn:oasis:names:tc:ciq:xsdschema:xAL:2.0" xmlns:ns15="http://www.w3.org/2005/Atom" xmlns:wps="http://www.opengis.net/wps/2.0" xmlns:ns14="http://www.w3.org/2001/SMIL20/" xmlns:wms="http://www.opengis.net/wms" xmlns:ns18="http://www.w3.org/2001/SMIL20/Language"> <ows:Identifier>621f40fd310b590001700589</ows:Identifier> <wps:Input id="Algorithm"> <wps:Data><wps:LiteralValue>FH</wps:LiteralValue></wps:Data> </wps:Input> <wps:Input id="bucket_name"> <wps:Data><wps:LiteralValue>maap-scientific-data</wps:LiteralValue></wps:Data> </wps:Input> <wps:Input id="s3_folder"> <wps:Data><wps:LiteralValue>shared/biopal-test-dataset/demo_lope_four</wps:LiteralValue></wps:Data> </wps:Input></ExecuteRequestType>
\ No newline at end of file