Skip to content
Snippets Groups Projects
Commit 77ec669d authored by jessica's avatar jessica
Browse files

BIOMASS-3126-Maap jupyter clean dockerfile and add librairies

parent 2a2e49f3
No related branches found
No related tags found
1 merge request!16BIOMASS-3126-Maap jupyter clean dockerfile and add librairies
%% Cell type:markdown id:1a70970d tags:
# <center> Tree Classification Demo </center>
%% Cell type:markdown id:bf3c2a51 tags:
<center>Tests for geopandas and gdal<center>
%% Cell type:markdown id:c55216cb tags:
## Import Packages
%% Cell type:code id:4d9291fa tags:
``` python
from datetime import datetime
import json
import time
import configparser
import requests
# On NASA , install with pip install geopandas
# On ESA; use a PolinSAR stack ( geopandas already embedded)
import geopandas as gpd
import pandas as pd
from creodias_finder import query
from shapely.geometry import Polygon,shape
```
%% Cell type:markdown id:711e5ecd tags:
## Define Functions
%% Cell type:code id:2b694047 tags:
``` python
def __get_result_S1( geometry_search, str_plateformeS1, str_product_type, str_sensor_mode, start_date, end_date):
'''
Recupere dans un dataframe les produits S1 qui intersectent la zone de recherche
:geometry_search: geometrie de la zone de recherche
:str_plateformeS1: nom de la plateforme
:str_product_type: nom du product type
:str_sensor_mode: nom du sensor mode
:start_date: date debut interval recherche au format datetime
:end_date: date fin interval recherche au format datetime
return: df_groupby
:rtype: dataframe
'''
# liste des produits de l'api creodias
results = query.query(
str_plateformeS1,
start_date=start_date,
end_date=end_date,
productType=str_product_type,
sensorMode=str_sensor_mode,
status="ONLINE",
geometry=geometry_search
)
# init des listes des parametres a conserver
list_title = []
list_date = []
list_orbit = []
list_orbitDirection = []
list_geom = []
list_url = []
# remplissage des listes avec les parametres des produits du results de l'api
for idproduct, dict_product in results.items():
list_title.append(dict_product['properties']['title'])
list_orbit.append(dict_product['properties']['relativeOrbitNumber'])
list_orbitDirection.append(dict_product['properties']['orbitDirection'])
list_geom.append(shape(dict_product['geometry']))
list_url.append(dict_product['properties']['productIdentifier'])
list_date.append(dict_product['properties']['title'][17:25])
# genere le gdf avec les produits s1
gdf_product = gpd.GeoDataFrame({'title' : list_title, 'date' : list_date, 'orbit' : list_orbit, 'direction' : list_orbitDirection,'geometry' : list_geom, 'url' : list_url}, crs = "4326")
# regroupe les produits qui ont les memes parametres de date, orbit et direction
df_groupby = gdf_product.groupby(['date', 'orbit', 'direction']).agg({'title' : list, 'url' : list, 'geometry' : list}).reset_index()
return df_groupby
```
%% Cell type:markdown id:2573ab84 tags:
## Define the study area or list of tile
%% Cell type:code id:7e14efe8 tags:
``` python
search_zone = [[4.751930185604931,45.37522170186796],[4.468812157001305,45.24092464324643],[4.018935832120382,44.88158298080388],[3.852035966569818,44.75014456444579],[3.898047329546759,44.56923769773521],[4.038870891160542,44.31329708744024],[4.241954580160842,44.26894961414283],[4.674279531570155,44.26246269728098],[4.882500597010154,44.8652889083827],[4.905098845459158,45.06277528749476],[4.782524828980907,45.368587240900965],[4.751930185604931,45.37522170186796]]
search_poly = Polygon(search_zone)
```
%% Cell type:markdown id:9b8ee178 tags:
## Search S1 products
%% Cell type:code id:f7fd2a1c tags:
``` python
# parametres de recherche
str_plateformeS1 = 'Sentinel1'
str_product_type = 'GRD'
str_sensor_mode = 'IW'
start_date = datetime(2022, 6, 3)
end_date = datetime(2023, 1, 31)
```
%% Cell type:code id:3cb6aa8f tags:
``` python
df_product = __get_result_S1( search_poly, str_plateformeS1, str_product_type, str_sensor_mode, start_date, end_date)
```
%% Output
query_url http://datahub.creodias.eu/resto/api/collections/Sentinel1/search.json?maxRecords=1000&startDate=2022-06-03T00%3A00%3A00&completionDate=2023-01-31T23%3A59%3A59&geometry=POLYGON+%28%284.751930185604931+45.37522170186796%2C+4.468812157001305+45.24092464324643%2C+4.018935832120382+44.88158298080388%2C+3.852035966569818+44.75014456444579%2C+3.898047329546759+44.56923769773521%2C+4.038870891160542+44.31329708744024%2C+4.241954580160842+44.26894961414283%2C+4.674279531570155+44.26246269728098%2C+4.882500597010154+44.8652889083827%2C+4.905098845459158+45.06277528749476%2C+4.782524828980907+45.368587240900965%2C+4.751930185604931+45.37522170186796%29%29&status=ONLINE&productType=GRD&sensorMode=IW
<Response [200]>
%% Cell type:code id:568856bd tags:
``` python
df_product
df_product = df_product[df_product['orbit']==37]
df_product
print(len(df_product))
```
%% Output
20
%% Cell type:markdown id:f85d0273 tags:
## Extract list of URL
%% Cell type:code id:6cfbf9c7 tags:
``` python
tmp_list_product_path = []
for prd in df_product['url']:
hour_1 = int(prd[0].split('/')[-1].split('_')[4].split('T')[1])
hour_2 = int(prd[1].split('/')[-1].split('_')[4].split('T')[1])
if hour_1 > hour_2:
tmp_list_product_path.append(prd[0])
else:
tmp_list_product_path.append(prd[1])
list_product_path = []
for path_url in tmp_list_product_path:
list_product_path.append(path_url[8:])
prod_payload = "[\""+",\"".join(list_product_path)+"\"]"
print(prod_payload)
```
%% Output
["Sentinel-1/SAR/GRD/2022/06/04/S1A_IW_GRDH_1SDV_20220604T055236_20220604T055301_043509_0531E5_DC1B.SAFE,"Sentinel-1/SAR/GRD/2022/06/16/S1A_IW_GRDH_1SDV_20220616T055236_20220616T055301_043684_05371A_E51A.SAFE,"Sentinel-1/SAR/GRD/2022/06/28/S1A_IW_GRDH_1SDV_20220628T055237_20220628T055302_043859_053C5F_3096.SAFE,"Sentinel-1/SAR/GRD/2022/07/22/S1A_IW_GRDH_1SDV_20220722T055238_20220722T055303_044209_0546C9_677E.SAFE,"Sentinel-1/SAR/GRD/2022/08/03/S1A_IW_GRDH_1SDV_20220803T055239_20220803T055304_044384_054BF3_4ACD.SAFE,"Sentinel-1/SAR/GRD/2022/08/15/S1A_IW_GRDH_1SDV_20220815T055240_20220815T055305_044559_05516B_B0D3.SAFE,"Sentinel-1/SAR/GRD/2022/08/27/S1A_IW_GRDH_1SDV_20220827T055240_20220827T055305_044734_055755_77D7.SAFE,"Sentinel-1/SAR/GRD/2022/09/08/S1A_IW_GRDH_1SDV_20220908T055241_20220908T055306_044909_055D33_8C27.SAFE,"Sentinel-1/SAR/GRD/2022/09/20/S1A_IW_GRDH_1SDV_20220920T055241_20220920T055306_045084_056320_8ED8.SAFE,"Sentinel-1/SAR/GRD/2022/10/02/S1A_IW_GRDH_1SDV_20221002T055242_20221002T055307_045259_0568FD_84DA.SAFE,"Sentinel-1/SAR/GRD/2022/10/14/S1A_IW_GRDH_1SDV_20221014T055242_20221014T055307_045434_056EDB_346E.SAFE,"Sentinel-1/SAR/GRD/2022/10/26/S1A_IW_GRDH_1SDV_20221026T055242_20221026T055307_045609_05740C_5826.SAFE,"Sentinel-1/SAR/GRD/2022/11/07/S1A_IW_GRDH_1SDV_20221107T055241_20221107T055306_045784_0579F9_649C.SAFE,"Sentinel-1/SAR/GRD/2022/11/19/S1A_IW_GRDH_1SDV_20221119T055241_20221119T055306_045959_057FDC_E997.SAFE,"Sentinel-1/SAR/GRD/2022/12/01/S1A_IW_GRDH_1SDV_20221201T055240_20221201T055305_046134_0585CE_836F.SAFE,"Sentinel-1/SAR/GRD/2022/12/13/S1A_IW_GRDH_1SDV_20221213T055240_20221213T055305_046309_058BCE_F90F.SAFE,"Sentinel-1/SAR/GRD/2022/12/25/S1A_IW_GRDH_1SDV_20221225T055239_20221225T055304_046484_0591C3_7C2C.SAFE,"Sentinel-1/SAR/GRD/2023/01/06/S1A_IW_GRDH_1SDV_20230106T055239_20230106T055304_046659_0597A7_7ED7.SAFE,"Sentinel-1/SAR/GRD/2023/01/18/S1A_IW_GRDH_1SDV_20230118T055238_20230118T055303_046834_059D93_5C5F.SAFE,"Sentinel-1/SAR/GRD/2023/01/30/S1A_IW_GRDH_1SDV_20230130T055238_20230130T055303_047009_05A37A_61B6.SAFE"]
%% Cell type:code id:0142ecbc tags:
``` python
## Test Gdal
```
%% Cell type:code id:ae1c9c71 tags:
``` python
!conda run -n maap sardem --bbox -156 18.8 -154.7 20.3 --data-source COP -o "/projects/treeclassificationdemo/driver/maap_utils/L1_MSI_pixel_value_Composite_123.tif" --output-format GTiff
```
%% Output
Creating output file that is 4680P x 5400L.
0...10...20...30...40...50...60...70...80...90...100 - done.
[02/28 16:51:44] [INFO dem.py] Bounds: -156.0 18.8 -154.7 20.3
[02/28 16:51:44] [INFO cop_dem.py] Creating /projects/treeclassificationdemo/driver/L1_MSI_pixel_value_Composite_123.tif
[02/28 16:51:44] [INFO cop_dem.py] Fetching remote tiles...
[02/28 16:51:44] [INFO cop_dem.py] Running GDAL command:
[02/28 16:51:44] [INFO cop_dem.py] gdalwarp /vsicurl/https://raw.githubusercontent.com/scottstanie/sardem/master/sardem/data/cop_global.vrt /projects/treeclassificationdemo/driver/L1_MSI_pixel_value_Composite_123.tif -of GTiff -ot Int16 -te -156 18.8000000000000007 -154.699999999999989 20.3000000000000007 -tr 0.000277777777777777778 0.000277777777777777778 -s_srs "epsg:4326+3855" -t_srs "epsg:4326" -wo NUM_THREADS=4 -r nearest -wm 5000 -multi
%% Cell type:markdown id:1a70970d tags:
# <center> Tree Classification Demo </center>
%% Cell type:markdown id:bf3c2a51 tags:
<center>Tests for geopandas and gdal<center>
%% Cell type:markdown id:c55216cb tags:
## Import Packages
%% Cell type:code id:4d9291fa tags:
``` python
from datetime import datetime
import json
import time
import configparser
import requests
# On NASA , install with pip install geopandas
# On ESA; use a PolinSAR stack ( geopandas already embedded)
import geopandas as gpd
import pandas as pd
from creodias_finder import query
from shapely.geometry import Polygon,shape
```
%% Cell type:markdown id:711e5ecd tags:
## Define Functions
%% Cell type:code id:2b694047 tags:
``` python
def __get_result_S1( geometry_search, str_plateformeS1, str_product_type, str_sensor_mode, start_date, end_date):
'''
Recupere dans un dataframe les produits S1 qui intersectent la zone de recherche
:geometry_search: geometrie de la zone de recherche
:str_plateformeS1: nom de la plateforme
:str_product_type: nom du product type
:str_sensor_mode: nom du sensor mode
:start_date: date debut interval recherche au format datetime
:end_date: date fin interval recherche au format datetime
return: df_groupby
:rtype: dataframe
'''
# liste des produits de l'api creodias
results = query.query(
str_plateformeS1,
start_date=start_date,
end_date=end_date,
productType=str_product_type,
sensorMode=str_sensor_mode,
status="ONLINE",
geometry=geometry_search
)
# init des listes des parametres a conserver
list_title = []
list_date = []
list_orbit = []
list_orbitDirection = []
list_geom = []
list_url = []
# remplissage des listes avec les parametres des produits du results de l'api
for idproduct, dict_product in results.items():
list_title.append(dict_product['properties']['title'])
list_orbit.append(dict_product['properties']['relativeOrbitNumber'])
list_orbitDirection.append(dict_product['properties']['orbitDirection'])
list_geom.append(shape(dict_product['geometry']))
list_url.append(dict_product['properties']['productIdentifier'])
list_date.append(dict_product['properties']['title'][17:25])
# genere le gdf avec les produits s1
gdf_product = gpd.GeoDataFrame({'title' : list_title, 'date' : list_date, 'orbit' : list_orbit, 'direction' : list_orbitDirection,'geometry' : list_geom, 'url' : list_url}, crs = "4326")
# regroupe les produits qui ont les memes parametres de date, orbit et direction
df_groupby = gdf_product.groupby(['date', 'orbit', 'direction']).agg({'title' : list, 'url' : list, 'geometry' : list}).reset_index()
return df_groupby
```
%% Cell type:markdown id:2573ab84 tags:
## Define the study area or list of tile
%% Cell type:code id:7e14efe8 tags:
``` python
search_zone = [[4.751930185604931,45.37522170186796],[4.468812157001305,45.24092464324643],[4.018935832120382,44.88158298080388],[3.852035966569818,44.75014456444579],[3.898047329546759,44.56923769773521],[4.038870891160542,44.31329708744024],[4.241954580160842,44.26894961414283],[4.674279531570155,44.26246269728098],[4.882500597010154,44.8652889083827],[4.905098845459158,45.06277528749476],[4.782524828980907,45.368587240900965],[4.751930185604931,45.37522170186796]]
search_poly = Polygon(search_zone)
```
%% Cell type:markdown id:9b8ee178 tags:
## Search S1 products
%% Cell type:code id:f7fd2a1c tags:
``` python
# parametres de recherche
str_plateformeS1 = 'Sentinel1'
str_product_type = 'GRD'
str_sensor_mode = 'IW'
start_date = datetime(2022, 6, 3)
end_date = datetime(2023, 1, 31)
```
%% Cell type:code id:3cb6aa8f tags:
``` python
df_product = __get_result_S1( search_poly, str_plateformeS1, str_product_type, str_sensor_mode, start_date, end_date)
```
%% Output
query_url http://datahub.creodias.eu/resto/api/collections/Sentinel1/search.json?maxRecords=1000&startDate=2022-06-03T00%3A00%3A00&completionDate=2023-01-31T23%3A59%3A59&geometry=POLYGON+%28%284.751930185604931+45.37522170186796%2C+4.468812157001305+45.24092464324643%2C+4.018935832120382+44.88158298080388%2C+3.852035966569818+44.75014456444579%2C+3.898047329546759+44.56923769773521%2C+4.038870891160542+44.31329708744024%2C+4.241954580160842+44.26894961414283%2C+4.674279531570155+44.26246269728098%2C+4.882500597010154+44.8652889083827%2C+4.905098845459158+45.06277528749476%2C+4.782524828980907+45.368587240900965%2C+4.751930185604931+45.37522170186796%29%29&status=ONLINE&productType=GRD&sensorMode=IW
<Response [200]>
%% Cell type:code id:568856bd tags:
``` python
df_product
df_product = df_product[df_product['orbit']==37]
df_product
print(len(df_product))
```
%% Output
20
%% Cell type:markdown id:f85d0273 tags:
## Extract list of URL
%% Cell type:code id:6cfbf9c7 tags:
``` python
tmp_list_product_path = []
for prd in df_product['url']:
hour_1 = int(prd[0].split('/')[-1].split('_')[4].split('T')[1])
hour_2 = int(prd[1].split('/')[-1].split('_')[4].split('T')[1])
if hour_1 > hour_2:
tmp_list_product_path.append(prd[0])
else:
tmp_list_product_path.append(prd[1])
list_product_path = []
for path_url in tmp_list_product_path:
list_product_path.append(path_url[8:])
prod_payload = "[\""+",\"".join(list_product_path)+"\"]"
print(prod_payload)
```
%% Output
["Sentinel-1/SAR/GRD/2022/06/04/S1A_IW_GRDH_1SDV_20220604T055236_20220604T055301_043509_0531E5_DC1B.SAFE,"Sentinel-1/SAR/GRD/2022/06/16/S1A_IW_GRDH_1SDV_20220616T055236_20220616T055301_043684_05371A_E51A.SAFE,"Sentinel-1/SAR/GRD/2022/06/28/S1A_IW_GRDH_1SDV_20220628T055237_20220628T055302_043859_053C5F_3096.SAFE,"Sentinel-1/SAR/GRD/2022/07/22/S1A_IW_GRDH_1SDV_20220722T055238_20220722T055303_044209_0546C9_677E.SAFE,"Sentinel-1/SAR/GRD/2022/08/03/S1A_IW_GRDH_1SDV_20220803T055239_20220803T055304_044384_054BF3_4ACD.SAFE,"Sentinel-1/SAR/GRD/2022/08/15/S1A_IW_GRDH_1SDV_20220815T055240_20220815T055305_044559_05516B_B0D3.SAFE,"Sentinel-1/SAR/GRD/2022/08/27/S1A_IW_GRDH_1SDV_20220827T055240_20220827T055305_044734_055755_77D7.SAFE,"Sentinel-1/SAR/GRD/2022/09/08/S1A_IW_GRDH_1SDV_20220908T055241_20220908T055306_044909_055D33_8C27.SAFE,"Sentinel-1/SAR/GRD/2022/09/20/S1A_IW_GRDH_1SDV_20220920T055241_20220920T055306_045084_056320_8ED8.SAFE,"Sentinel-1/SAR/GRD/2022/10/02/S1A_IW_GRDH_1SDV_20221002T055242_20221002T055307_045259_0568FD_84DA.SAFE,"Sentinel-1/SAR/GRD/2022/10/14/S1A_IW_GRDH_1SDV_20221014T055242_20221014T055307_045434_056EDB_346E.SAFE,"Sentinel-1/SAR/GRD/2022/10/26/S1A_IW_GRDH_1SDV_20221026T055242_20221026T055307_045609_05740C_5826.SAFE,"Sentinel-1/SAR/GRD/2022/11/07/S1A_IW_GRDH_1SDV_20221107T055241_20221107T055306_045784_0579F9_649C.SAFE,"Sentinel-1/SAR/GRD/2022/11/19/S1A_IW_GRDH_1SDV_20221119T055241_20221119T055306_045959_057FDC_E997.SAFE,"Sentinel-1/SAR/GRD/2022/12/01/S1A_IW_GRDH_1SDV_20221201T055240_20221201T055305_046134_0585CE_836F.SAFE,"Sentinel-1/SAR/GRD/2022/12/13/S1A_IW_GRDH_1SDV_20221213T055240_20221213T055305_046309_058BCE_F90F.SAFE,"Sentinel-1/SAR/GRD/2022/12/25/S1A_IW_GRDH_1SDV_20221225T055239_20221225T055304_046484_0591C3_7C2C.SAFE,"Sentinel-1/SAR/GRD/2023/01/06/S1A_IW_GRDH_1SDV_20230106T055239_20230106T055304_046659_0597A7_7ED7.SAFE,"Sentinel-1/SAR/GRD/2023/01/18/S1A_IW_GRDH_1SDV_20230118T055238_20230118T055303_046834_059D93_5C5F.SAFE,"Sentinel-1/SAR/GRD/2023/01/30/S1A_IW_GRDH_1SDV_20230130T055238_20230130T055303_047009_05A37A_61B6.SAFE"]
%% Cell type:code id:0142ecbc tags:
``` python
## Test Gdal
```
%% Cell type:code id:ae1c9c71 tags:
``` python
!conda run -n maap sardem --bbox -156 18.8 -154.7 20.3 --data-source COP -o "/projects/treeclassificationdemo/driver/maap_utils/L1_MSI_pixel_value_Composite_123.tif" --output-format GTiff
```
%% Output
Creating output file that is 4680P x 5400L.
0...10...20...30...40...50...60...70...80...90...100 - done.
[02/28 16:51:44] [INFO dem.py] Bounds: -156.0 18.8 -154.7 20.3
[02/28 16:51:44] [INFO cop_dem.py] Creating /projects/treeclassificationdemo/driver/L1_MSI_pixel_value_Composite_123.tif
[02/28 16:51:44] [INFO cop_dem.py] Fetching remote tiles...
[02/28 16:51:44] [INFO cop_dem.py] Running GDAL command:
[02/28 16:51:44] [INFO cop_dem.py] gdalwarp /vsicurl/https://raw.githubusercontent.com/scottstanie/sardem/master/sardem/data/cop_global.vrt /projects/treeclassificationdemo/driver/L1_MSI_pixel_value_Composite_123.tif -of GTiff -ot Int16 -te -156 18.8000000000000007 -154.699999999999989 20.3000000000000007 -tr 0.000277777777777777778 0.000277777777777777778 -s_srs "epsg:4326+3855" -t_srs "epsg:4326" -wo NUM_THREADS=4 -r nearest -wm 5000 -multi
File added
File added
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment