Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
#!/usr/local/bin/python
import sys, getopt
import requests
import json
import sys
import os
import time
import math
import os.path as path
# Import getopt module
import getopt
MAAP_ENV_TYPE = os.getenv("MAAP_ENV_TYPE")
CLIENT_ID = os.getenv("CLIENT_ID")
BEARER=""
USER_INFO_FILE_PATH="/usr/bmap/maap-s3-userinfo.json"
userinfo = {}
def display_help():
print('Usage: [option...] {-f|-u|-d|-l|-r}')
#print('-i Get a fresh token before any request. It ask for email and password')
print('-u myFile.tiff locally path/myFile.tiff in the S3 Upload data in the S3')
print('-d path/in/S3/file.tiff myFileName.tiff Download a data from the S3')
print('-l folder/path List data in a subfolder')
print('-r path/in/S3/file.tiff Delete an existing data on S3')
print('-f refresh credentials and password')
sys.exit(2)
#########################
# Init the bearer #
#########################
def init():
if os.path.isfile(USER_INFO_FILE_PATH):
print("[INFO] Personal user info is find")
#Check if the file is created less than one hour
if is_file_older_than_x_hour(USER_INFO_FILE_PATH):
print("[INFO] Token is expired, we generate a new one")
#Get the email and password info
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
email=userinfo['email']
password=userinfo['password']
#Regenerate token
#Function to generate a new token
generate_token(email, password)
else:
print("[INFO] Token is still valid")
else:
print("[INFO] Personal user info is not found")
refresh()
###########################
# Refresh token and save #
###########################
def refresh():
email = input("Your email: ")
#password
password = input("Your password: ")
#Function to generate a new token
generate_token(email, password)
###########################
# Generate token and save #
###########################
def generate_token(email, password):
print("[INFO] Start retrieving token for authent")
#Set the bearer
url = "https://iam."+MAAP_ENV_TYPE.lower()+".esa-maap.org/oxauth/restv1/token"
print (url)
print (CLIENT_ID)
response = requests.post(url, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
print(response)
#Convert the string to json to fecth access_token
data = json.loads(response.text)
token = data['access_token']
# add the token in the json info file
#Create a json with email and password
userinfo = {
'email': email,
'password': password,
'token': token
}
if token:
#add the json in the file
with open(USER_INFO_FILE_PATH, 'w') as outfile:
json.dump(userinfo, outfile)
print("[INFO] Token saved for one hour and ready to be used "+token)
else:
print("[ERROR] Token is empty. Please 1) run refresh (-r) function and check your password")
# Terminate the script
sys.exit(2)
#########################
# Check if file is older#
#########################
def is_file_older_than_x_hour(file, hour=1):
file_time = path.getmtime(file)
# Check against 214 hour
return ((time.time() - file_time) > 3600*hour)
#########################
# Upload the data in S3 #
#########################
def upload(sourceFile, destination):
print("[INFO] Source file is : ", sourceFile)
print("[INFO] Destination file is : ", destination)
if sourceFile and destination:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
# If the file is less that 100 MB we upload directly
#Check file size
fileSize = os.stat(sourceFile).st_size
print("Size "+ str(fileSize))
#We have more than 5GB
if fileSize > 5000000000:
#We upload the multi part data
print("[INFO] Starting multi part upload")
upload_multipart(sourceFile, destination)
else:
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
print("[INFO] Starting retrieving the presigned url for the creation of the file with token "+ token)
#files = {'upload_file': open(sourceFile,'rb')}
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
response = requests.put(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
print("[INFO] Location is "+ location)
if location:
print("[INFO] Start uploading the file")
with open(sourceFile, 'rb') as f:
response = requests.put(location, data=f)
print(response)
#files = {'file': open(sourceFile, 'rb')}
#r = requests.put(location, files=files)
else:
print("[ERROR] Presigned url not generated. Please re run refresh or contact admin if the error persist")
else:
display_help()
###########################################################
# Upload the data in S3, the data is split chunk by chunk #
###########################################################
def upload_multipart(sourceFile, destination):
#Get the token
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Set variables
filePath = sourceFile
key = destination
fileSize = os.stat(filePath).st_size
print("Size "+ str(fileSize))
#Set max to split to 5M
max_size = 5 * 1024 * 1024 # Approach 1: Assign the size
nbParts = math.ceil(fileSize/max_size) #calculate nbParts
print("[INFO] We will have "+ str(nbParts)+" parts")
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateUploadId"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
print("[INFO] uploadId "+ response.text)
#Save upload id
uploadId = response.text
#Generate presigned urls
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateListPresignedUrls"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
stringList = response.text
str1 = stringList.replace(']','').replace('[','')
listPresignedUrl = str1.replace('"','').split(",")
# we load the data
print(listPresignedUrl)
parts = []
#sys.stdout = open("log.txt", "w")
with open(filePath, 'rb') as f:
i = 0
while i < nbParts:
print("Upload part "+ str(i))
file_data = f.read(max_size)
headers={'Content-Length': str(max_size)}
#print(listPresignedUrl[i])
response = requests.put(listPresignedUrl[i], data=file_data, headers=headers)
#print(response.headers)
#print(response.text)
etag = response.headers['ETag']
parts.append({'eTag': etag, 'partNumber': int(i+1)})
print(parts)
i = i+1
#sys.stdout.close()
#complete the multi part
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
###################
# Delete the data #
####################
def delete(destination):
print("[INFO] Destination file is : ", destination)
if destination:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#call the api to delete the data
#Get the presigned url to delete the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
response = requests.delete(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
#We have the
if location:
#We delete the data using the location
print("[INFO] we are about to delete")
response = requests.delete(location)
print(response)
else:
display_help()
###################
# download the data #
####################
def download(path, name):
print("[INFO] path file is : ", path)
if path:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Get the presigned url to download the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+path
response = requests.get(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
#We have the
if location:
#We download the data using the location
print("[INFO] we are about to download the data")
response = requests.get(location)
open(name, 'wb').write(response.content)
print("[INFO] Download finished")
else:
display_help()
##########################
# list data in s3 folder #
##########################
def list(path):
print("[INFO]: Start finding data in path : "+path)
if path:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#call the api to delete the data
#Get the presigned url to delete the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+path+"?list=true"
response = requests.get(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
print("[INFO] Result list:")
if(response.text):
print(response.text)
else:
print("[INFO] No data found")
else:
display_help()
# Store argument variable omitting the script name
argv = sys.argv[1:]
# Initialize result variable
result=0
try:
# Define getopt short and long options
options, args = getopt.getopt(sys.argv[1:], 'u:r:l:d:i:f:h', ['upload=', 'remove=','list=','download=','init=','refresh=','help='])
# Read each option using for loop
for opt, arg in options:
# Calculate the sum if the option is -a or --add
if opt in ('-u', '--upload'):
# Upload a data
if len(argv) != 3:
display_help()
else:
upload(argv[1], argv[2])
# Delete the data
elif opt in ('-r', '--remove'):
# Delete a data
if len(argv) != 2:
display_help()
else:
delete(argv[1])
elif opt in ('-l', '--list'):
# list a folder
if len(argv) != 2:
display_help()
else:
list(argv[1])
elif opt in ('-d', '--download'):
# Download a data
if len(argv) != 3:
display_help()
else:
download(argv[1], argv[2])
elif opt in ('-i', '--init'):
# Download a data
init()
elif opt in ('-f', '--refresh'):
# Download a data
refresh()
elif opt in ('-h', '--help'):
# Print the option
display_help()
except getopt.GetoptError:
# Print the error message if the wrong option is provided
print('The wrong option is provided. Please run -h')
# Terminate the script
sys.exit(2)