check in AWS Lambda code used for WebKitArchiveSupport
authorlforschler@apple.com <lforschler@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 28 Sep 2017 21:43:43 +0000 (21:43 +0000)
committerlforschler@apple.com <lforschler@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 28 Sep 2017 21:43:43 +0000 (21:43 +0000)
https://bugs.webkit.org/show_bug.cgi?id=177614

Rubber-stamped by Alexey Proskuryakov.

* WebKitArchiveSupport/lambda: Added.
* WebKitArchiveSupport/lambda/delete-minified-s3-archive-from-dynamodb.py: Added.
(lambda_handler):
* WebKitArchiveSupport/lambda/delete-s3-archive-from-dynamodb.py: Added.
(lambda_handler):
* WebKitArchiveSupport/lambda/register-archive-in-dynamodb.py: Added.
(lambda_handler):
* WebKitArchiveSupport/lambda/register-minified-s3-archive-in-dynamodb.py: Added.
(lambda_handler):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@222634 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/WebKitArchiveSupport/lambda/delete-minified-s3-archive-from-dynamodb.py [new file with mode: 0644]
Tools/WebKitArchiveSupport/lambda/delete-s3-archive-from-dynamodb.py [new file with mode: 0644]
Tools/WebKitArchiveSupport/lambda/register-archive-in-dynamodb.py [new file with mode: 0644]
Tools/WebKitArchiveSupport/lambda/register-minified-s3-archive-in-dynamodb.py [new file with mode: 0644]

index 4443c8b..c1e69cf 100644 (file)
@@ -1,3 +1,20 @@
+2017-09-28  Lucas Forschler  <lforschler@apple.com>
+
+        check in AWS Lambda code used for WebKitArchiveSupport
+        https://bugs.webkit.org/show_bug.cgi?id=177614
+        
+        Rubber-stamped by Alexey Proskuryakov.
+
+        * WebKitArchiveSupport/lambda: Added.
+        * WebKitArchiveSupport/lambda/delete-minified-s3-archive-from-dynamodb.py: Added.
+        (lambda_handler):
+        * WebKitArchiveSupport/lambda/delete-s3-archive-from-dynamodb.py: Added.
+        (lambda_handler):
+        * WebKitArchiveSupport/lambda/register-archive-in-dynamodb.py: Added.
+        (lambda_handler):
+        * WebKitArchiveSupport/lambda/register-minified-s3-archive-in-dynamodb.py: Added.
+        (lambda_handler):
+
 2017-09-28  Jiewen Tan  <jiewen_tan@apple.com>
 
         WeakPtrFactory should allow downcasting
diff --git a/Tools/WebKitArchiveSupport/lambda/delete-minified-s3-archive-from-dynamodb.py b/Tools/WebKitArchiveSupport/lambda/delete-minified-s3-archive-from-dynamodb.py
new file mode 100644 (file)
index 0000000..a1b7ee1
--- /dev/null
@@ -0,0 +1,32 @@
+import boto3
+import json
+import urllib.parse  # pylint: disable=E0611
+
+print('Loading function')
+dynamodb_client = boto3.client('dynamodb')
+table_name = 'minified-archives.webkit.org'
+s3 = boto3.client('s3')
+
+
+def lambda_handler(event, context):
+    print("Received event: " + json.dumps(event, indent=2))
+
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')  # pylint: disable=E1101
+    # ex: mac-sierra-x86_64-debug/218331.zip
+
+    split_key = key.split('/')
+    identifier = split_key[0]           # mac-sierra-x86_64-debug
+    filename = split_key[1]             # 218331.zip
+    revision = filename.split('.')[0]   # 218331
+
+    print ("identifier: " + identifier)
+    print ("revision: " + revision)
+
+    try:
+        item = {'identifier': {'S': identifier}, 'revision': {'N': revision}}
+        response = dynamodb_client.delete_item(TableName=table_name, Key=item)
+        return response
+    except Exception as e:
+        print(e)
+        print('Error deleting item: {}\nfrom database: {}.'.format(item, table_name))
+        raise e
diff --git a/Tools/WebKitArchiveSupport/lambda/delete-s3-archive-from-dynamodb.py b/Tools/WebKitArchiveSupport/lambda/delete-s3-archive-from-dynamodb.py
new file mode 100644 (file)
index 0000000..63a0ee9
--- /dev/null
@@ -0,0 +1,32 @@
+import boto3
+import json
+import urllib.parse  # pylint: disable=E0611
+
+print('Loading function')
+dynamodb_client = boto3.client('dynamodb')
+table_name = 'archives.webkit.org'
+s3 = boto3.client('s3')
+
+
+def lambda_handler(event, context):
+    print("Received event: " + json.dumps(event, indent=2))
+
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')  # pylint: disable=E1101
+    # ex: mac-sierra-x86_64-debug/218331.zip
+
+    split_key = key.split('/')
+    identifier = split_key[0]           # mac-sierra-x86_64-debug
+    filename = split_key[1]             # 218331.zip
+    revision = filename.split('.')[0]   # 218331
+
+    print ("identifier: " + identifier)
+    print ("revision: " + revision)
+
+    try:
+        item = {'identifier': {'S': identifier}, 'revision': {'N': revision}}
+        response = dynamodb_client.delete_item(TableName=table_name, Key=item)
+        return response
+    except Exception as e:
+        print(e)
+        print('Error deleting item: {}\nfrom database: {}.'.format(item, table_name))
+        raise e
diff --git a/Tools/WebKitArchiveSupport/lambda/register-archive-in-dynamodb.py b/Tools/WebKitArchiveSupport/lambda/register-archive-in-dynamodb.py
new file mode 100644 (file)
index 0000000..0b4acad
--- /dev/null
@@ -0,0 +1,48 @@
+import boto3
+import json
+import time
+import urllib.parse  # pylint: disable=E0611
+
+DAYS_TO_KEEP = 14
+EPOCH_DAY = 86400
+TABLE_NAME = 'archives.webkit.org'
+URL_PREFIX = 'https://s3-us-west-2.amazonaws.com'
+
+dynamodb_client = boto3.client('dynamodb')
+s3 = boto3.client('s3')
+
+
+def lambda_handler(event, context):
+    print("Received event: " + json.dumps(event, indent=2))
+
+    # Get the bucket name from the event
+    bucket = event['Records'][0]['s3']['bucket']['name']
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')  # pylint: disable=E1101
+    # ex: mac-sierra-x86_64-debug/218331.zip
+
+    split_key = key.split('/')
+    identifier = split_key[0]           # mac-sierra-x86_64-debug
+    filename = split_key[1]             # 218331.zip
+    revision = filename.split('.')[0]   # 218331
+    creation_time = int(time.time())
+    expiration_time = creation_time + DAYS_TO_KEEP * EPOCH_DAY
+
+    creation_time = str(creation_time)
+    expiration_time = str(expiration_time)
+    s3_url = '/'.join([URL_PREFIX, bucket, key])
+
+    try:
+        item = {
+            'identifier': {'S': identifier},
+            'revision': {'N': revision},
+            's3_url': {'S': s3_url},
+            'creationTime': {'N': creation_time},
+            'expirationTime': {'N': expiration_time},
+        }
+        print('Item: {}'.format(item))
+        dynamodb_client.put_item(TableName=TABLE_NAME, Item=item)
+        return s3_url
+    except Exception as e:
+        print(e)
+        print('Error registering item: {}\nfrom bucket {}.'.format(item, bucket))
+        raise e
diff --git a/Tools/WebKitArchiveSupport/lambda/register-minified-s3-archive-in-dynamodb.py b/Tools/WebKitArchiveSupport/lambda/register-minified-s3-archive-in-dynamodb.py
new file mode 100644 (file)
index 0000000..c0be5d6
--- /dev/null
@@ -0,0 +1,48 @@
+import boto3
+import json
+import time
+import urllib.parse  # pylint: disable=E0611
+
+DAYS_TO_KEEP = 910
+EPOCH_DAY = 86400
+TABLE_NAME = 'minified-archives.webkit.org'
+URL_PREFIX = 'https://s3-us-west-2.amazonaws.com'
+
+dynamodb_client = boto3.client('dynamodb')
+s3 = boto3.client('s3')
+
+
+def lambda_handler(event, context):
+    print("Received event: " + json.dumps(event, indent=2))
+
+    # Get the bucket name from the event
+    bucket = event['Records'][0]['s3']['bucket']['name']
+    key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'], encoding='utf-8')  # pylint: disable=E1101
+    # ex: mac-sierra-x86_64-debug/218331.zip
+
+    split_key = key.split('/')
+    identifier = split_key[0]           # mac-sierra-x86_64-debug
+    filename = split_key[1]             # 218331.zip
+    revision = filename.split('.')[0]   # 218331
+    creation_time = int(time.time())
+    expiration_time = creation_time + DAYS_TO_KEEP * EPOCH_DAY
+
+    creation_time = str(creation_time)
+    expiration_time = str(expiration_time)
+    s3_url = '/'.join([URL_PREFIX, bucket, key])
+
+    try:
+        item = {
+            'identifier': {'S': identifier},
+            'revision': {'N': revision},
+            's3_url': {'S': s3_url},
+            'creationTime': {'N': creation_time},
+            'expirationTime': {'N': expiration_time},
+        }
+        print('Item: {}'.format(item))
+        dynamodb_client.put_item(TableName=TABLE_NAME, Item=item)
+        return s3_url
+    except Exception as e:
+        print(e)
+        print('Error registering item: {}\nfrom bucket {}.'.format(item, bucket))
+        raise e