Newer
Older
#!/usr/local/bin/python
import sys, getopt
import requests
import json
import sys
import os
import time
import math
import os.path as path
# Import getopt module
import getopt
MAAP_ENV_TYPE = os.getenv("MAAP_ENV_TYPE")
CLIENT_ID = os.getenv("CLIENT_ID")
BEARER=""
#if windows we take the current folder
if sys.platform == 'win32':
USER_INFO_FILE_PATH=os.getcwd()+"\maap-s3-userinfo.json"
USER_LAST_UPLOAD_INFO_FILE_PATH=os.getcwd()+"\maap-s3-multipartinfo.json"
else :
USER_INFO_FILE_PATH="/usr/bmap/maap-s3-userinfo.json"
USER_LAST_UPLOAD_INFO_FILE_PATH="/usr/bmap/maap-s3-multipartinfo.json"
multipartinfo = {}
print('Usage: [option...] {upload|download|list|delete|refresh|resume}')
#print('-i Get a fresh token before any request. It ask for email and password')
print('upload myFile.tiff locally path/myFile.tiff in the S3 Upload data in the S3')
print('download myFileName.tiff path/in/S3/file.tiff Download a data from the S3')
print('list folder/path List data in a subfolder')
print('delete path/in/S3/file.tiff Delete an existing data on S3')
print('refresh Refresh credentials and password')
print('resume Resume last interrupted multipart upload')
sys.exit(2)
#########################
# Init the bearer #
#########################
def init():
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
if os.path.isfile(USER_INFO_FILE_PATH):
print("[INFO] Personal user info is find")
#Check if the file is created less than one hour
if is_file_older_than_x_hour(USER_INFO_FILE_PATH):
print("[INFO] Token is expired, we generate a new one")
#Get the email and password info
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
email=userinfo['email']
password=userinfo['password']
#Regenerate token
#Function to generate a new token
generate_token(email, password)
else:
print("[INFO] Token is still valid")
else:
print("[INFO] Personal user info is not found")
refresh()
###########################
# Refresh token and save #
###########################
def refresh():
email = input("Your email: ")
#password
password = input("Your password: ")
#Function to generate a new token
generate_token(email, password)
###########################
# Generate token and save #
###########################
def generate_token(email, password):
print("[INFO] Start retrieving token for authent")
#Set the bearer
url = "https://iam."+MAAP_ENV_TYPE.lower()+".esa-maap.org/oxauth/restv1/token"
print (url)
print (CLIENT_ID)
response = requests.post(url, data={'client_id': CLIENT_ID, 'username': email, 'password': password, "grant_type": "password", "scope": "openid+profile"})
print(response)
#Convert the string to json to fecth access_token
data = json.loads(response.text)
token = data['access_token']
# add the token in the json info file
#Create a json with email and password
userinfo = {
'email': email,
'password': password,
'token': token
}
if token:
#add the json in the file
with open(USER_INFO_FILE_PATH, 'w') as outfile:
json.dump(userinfo, outfile)
print("[INFO] Token saved for one hour and ready to be used "+token)
else:
print("[ERROR] Token is empty. Please 1) run refresh (-r) function and check your password")
# Terminate the script
sys.exit(2)
#########################
# Check if file is older#
#########################
def is_file_older_than_x_hour(file, hour=1):
file_time = path.getmtime(file)
# Check against 214 hour
return ((time.time() - file_time) > 3600*hour)
#########################
# Upload the data in S3 #
#########################
def upload(sourceFile, destination):
print("[INFO] Source file is : ", sourceFile)
print("[INFO] Destination file is : ", destination)
if sourceFile and destination:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
# If the file is less that 100 MB we upload directly
#Check file size
fileSize = os.stat(sourceFile).st_size
print("Size "+ str(fileSize))
#We have more than 5GB
if fileSize > 5000000000:
#We upload the multi part data
print("[INFO] Starting multi part upload")
upload_multipart(sourceFile, destination)
else:
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
print("[INFO] Starting retrieving the presigned url for the creation of the file with token "+ token)
#files = {'upload_file': open(sourceFile,'rb')}
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
response = requests.put(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
print("[INFO] Location is "+ location)
if location:
print("[INFO] Start uploading the file")
with open(sourceFile, 'rb') as f:
response = requests.put(location, data=f)
print(response)
#files = {'file': open(sourceFile, 'rb')}
#r = requests.put(location, files=files)
else:
print("[ERROR] Presigned url not generated. Please re run refresh or contact admin if the error persist")
else:
display_help()
###########################################################
# Upload the data in S3, the data is split chunk by chunk #
###########################################################
def upload_multipart(sourceFile, destination):
#Get the token
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Set variables
filePath = sourceFile
key = destination
fileSize = os.stat(filePath).st_size
print("Size "+ str(fileSize))
#Set max to split to 5M
max_size = 5 * 1024 * 1024 # Approach 1: Assign the size
nbParts = math.ceil(fileSize/max_size) #calculate nbParts
print("[INFO] We will have "+ str(nbParts)+" parts")
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateUploadId"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
print("[INFO] uploadId "+ response.text)
#Save upload id
uploadId = response.text
#Generate presigned urls
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateListPresignedUrls"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
stringList = response.text
str1 = stringList.replace(']','').replace('[','')
listPresignedUrl = str1.replace('"','').split(",")
# we load the data
#print(listPresignedUrl)
parts = []
#sys.stdout = open("log.txt", "w")
with open(filePath, 'rb') as f:
i = 0
while i < nbParts:
print("Upload part "+ str(i))
file_data = f.read(max_size)
headers={'Content-Length': str(max_size)}
#print(listPresignedUrl[i])
response = requests.put(listPresignedUrl[i], data=file_data, headers=headers)
#print(response.headers)
#print(response.text)
etag = response.headers['ETag']
parts.append({'eTag': etag, 'partNumber': int(i+1)})
print(parts)
i = i+1
#We save also the multipart
#So we can resume it if upload failed
multipartinfo = {
'uploadId': uploadId,
'partsUpploaded': parts,
'sourceFile': sourceFile,
'destination': destination
}
#add the json in the file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH, 'w') as outfile:
json.dump(multipartinfo, outfile)
#sys.stdout.close()
#complete the multi part
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
#delete the file of multipart info because upload was success
os.remove(USER_LAST_UPLOAD_INFO_FILE_PATH)
###################################
# Resume failed multi part upload #
###################################
def resume():
print("[INFO] Resume the last multipart upload")
print("[INFO] Check last multipart upload metadata")
if os.path.isfile(USER_LAST_UPLOAD_INFO_FILE_PATH):
#Generate or get a token
init()
print("[INFO] Previous multi part upload file found")
token=''
#Get the token
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Get the data in the json file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH) as json_file:
multipartinfo = json.load(json_file)
#Get the info
uploadId=multipartinfo['uploadId']
destination=multipartinfo['destination']
sourceFile=multipartinfo['sourceFile']
partsUpploaded=multipartinfo['partsUpploaded']
#We get the presigned url
fileSize = os.stat(sourceFile).st_size
print("Size "+ str(fileSize))
#Set max to split to 5M
max_size = 5 * 1024 * 1024 # Approach 1: Assign the size
nbParts = math.ceil(fileSize/max_size) #calculate nbParts
finalPart = nbParts - len(partsUpploaded)
print("[INFO] We will have "+ str(nbParts)+" parts minus already uploaded parts. We have to push "+ str(finalPart) +" parts")
#Generate presigned urls
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/generateListPresignedUrls"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': destination, 'nbParts': finalPart, 'uploadId': uploadId}
response = requests.get(url, params = params, headers = {'Authorization': 'Bearer '+token})
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
stringList = response.text
str1 = stringList.replace(']','').replace('[','')
listPresignedUrl = str1.replace('"','').split(",")
#Get the json uploaded list
#we iterate over the data and repush
with open(sourceFile, 'rb') as f:
i = 0
presignedUrlIndex = 0
while i < nbParts:
file_data = f.read(max_size)
#We upload only if we have new nn uploaded part
if i > len(partsUpploaded):
print("Upload part "+ str(i))
headers={'Content-Length': str(max_size)}
#print(listPresignedUrl[i])
response = requests.put(listPresignedUrl[presignedUrlIndex], data=file_data, headers=headers)
#print(response.headers)
#print(response.text)
etag = response.headers['ETag']
partsUpploaded.append({'eTag': etag, 'partNumber': int(i+1)})
presignedUrlIndex = presignedUrlIndex + 1
#We save also the multipart
#So we can resume it if upload failed
multipartinfo = {
'uploadId': uploadId,
'partsUpploaded': partsUpploaded,
'sourceFile': sourceFile,
'destination': destination
}
#add the json in the file
with open(USER_LAST_UPLOAD_INFO_FILE_PATH, 'w') as outfile:
json.dump(multipartinfo, outfile)
#We increase the data
i = i+1
#complete the multi part
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/completeMultiPartUploadRequest"
params={'bucketName': 'bmap-catalogue-data', 'objectKey': key, 'nbParts': nbParts, 'uploadId': uploadId}
response = requests.get(url, data=str(parts), params = params, headers = {'Authorization': 'Bearer '+token})
#delete the file of multipart info because upload was success
os.remove(USER_LAST_UPLOAD_INFO_FILE_PATH)
else:
print("[INFO] Please run upload. There are no upload to be resume")
###################
# Delete the data #
####################
def delete(destination):
print("[INFO] Destination file is : ", destination)
if destination:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#call the api to delete the data
#Get the presigned url to delete the data
url = "http://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+destination
print(url)
response = requests.delete(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
location = response.headers['Location']
#We have the
if location:
#We delete the data using the location
print("[INFO] we are about to delete")
response = requests.delete(location)
print(response)
else:
display_help()
###################
# download the data #
####################
def download(path, name):
print("[INFO] path file is : ", path)
if path:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#Get the presigned url to download the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+path
response = requests.get(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
location = response.headers['Location']
#We have the
if location:
#We download the data using the location
print("[INFO] we are about to download the data")
download_file(location, name)
#response = requests.get(location)
#open(name, 'wb').write(response.content)
print("[INFO] Download finished")
else:
display_help()
##########################
# download file using url #
##########################
def download_file(url, name):
#local_filename = url.split('/')[-1]
# NOTE the stream=True parameter below
with requests.get(url, stream=True) as r:
r.raise_for_status()
with open(name, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
# If you have chunk encoded response uncomment if
# and set chunk_size parameter to None.
#if chunk:
f.write(chunk)
return name
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
##########################
# list data in s3 folder #
##########################
def list(path):
print("[INFO]: Start finding data in path : "+path)
if path:
print("[INFO] Get an existing or fresh token")
#Generate or get a token
init()
with open(USER_INFO_FILE_PATH) as json_file:
userinfo = json.load(json_file)
#Get the info
token=userinfo['token']
#call the api to delete the data
#Get the presigned url to delete the data
url = "https://gravitee-gateway."+MAAP_ENV_TYPE.lower()+".esa-maap.org/s3/"+path+"?list=true"
response = requests.get(url, headers = {'Authorization': 'Bearer '+token}, allow_redirects=False)
print("[INFO] Result list:")
if(response.text):
print(response.text)
else:
print("[INFO] No data found")
else:
display_help()
# Store argument variable omitting the script name
argv = sys.argv[1:]
# Initialize result variable
result=0
try:
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
if len(argv) == 0:
display_help()
else:
if argv[0] == 'resume':
resume()
elif argv[0] == 'refresh':
refresh()
elif argv[0] == 'upload':
# Upload a data
if len(argv) != 3:
display_help()
else:
upload(argv[1], argv[2])
elif argv[0] == 'delete':
# Delete a data
if len(argv) != 2:
display_help()
else:
delete(argv[1])
elif argv[0] == 'download':
# Download a data
if len(argv) != 3:
display_help()
else:
download(argv[1], argv[2])
elif argv[0] == 'list':
# list a folder
if len(argv) != 2:
display_help()
else:
list(argv[1])
elif argv[0] == 'help':