Snapshot 4n6ir Imager for Docker

THE CHALLENGE

Snapshot 4n6ir Imager python script has worked great for converting Amazon EBS Snapshots to DD images! A containerized version was needed so that the imaging process could be automated. The new version needs to support the incident response playbook with the following features:

  • Secure Transfer
  • High Availability
  • Encryption Management
  • Temporary Credentials
  • Cost-Conscious
SNAPSHOT 4N6IR IMAGER FOR DOCKER
Snapshot 4n6ir Imager for Docker v0.1.1

optional arguments:
  -h, --help           show this help message and exit

Required:
  --region REGION      us-east-2
  --snapshot SNAPSHOT  snap-056e0b1bd07ad91b2
  --token TOKEN        abacadaba-abacadaba-abacadaba

The end-user initially receives an email with a token to access the Upload API for a specific region. Remember to store the API Key in AWS Secrets Manager or AWS Systems Manager Parameter Store to protect the credential.

Cloud 4n6ir Upload API Key

	URL Link: https://upload.us-east-2.4n6ir.com
	API Key: abacadaba-abacadaba-abacadaba
	AWS Region: us-east-2
	TTL Seconds: 120

Accessing the Upload API generates a pre-signed URL with a short TTL to an S3 bucket in the specific region for each block. Block size is only 512K by default that gets GZIP compressed, well under the 5 GB file size limit for a single S3 put object call. The snapshot block encrypts with the auto-generated keys before the transfer from the Upload API response.

$ python3 Snapshot-4n6ir-Imager-for-Docker.py --region us-east-2 --snapshot snap-07fd2195ff4777cfe --token abacadaba-abacadaba-abacadaba

Snapshot 4n6ir Imager for Docker v0.1.1

	Region: 	us-east-2
	Snapshot: 	snap-090e77f6aabdf5435
	Blocks: 	2730
	Completed: 	Confirmed!

The primary objective is security, but the cost needs to be considered and accomplished by limiting data transfers to regional with the pre-signed S3 URLs. Once the image uploads to an S3 bucket, storage costs are less than EBS Snapshots plus opens the opportunity to use Amazon Glacier for additional cost savings.

Automation plays such a critical part in helping incident handlers deal with the volume of events that they need to respond too! Figured I would share an early iteration as I work on a Snapshot 4n6ir Pipeline for AWS.

Happy Coding,

John Lukach

SOURCE CODE
import argparse
import base64
import boto3
import gzip
import hashlib
import io
import json
import os
import requests
import shutil
from cryptography.fernet import Fernet
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC

def createkey(password,salt):
	kdf = PBKDF2HMAC(algorithm=hashes.SHA256(),
					 length=32,
					 salt=salt.encode(),
					 iterations=100000,
					 backend=default_backend())
	key = base64.urlsafe_b64encode(kdf.derive(password.encode()))
	f = Fernet(key)
	return f
	
def completed(region,snapshot,token,count):
	object_name = region+'|'+snapshot+'|'+str(count)
	link = 'https://upload.'+region+'.4n6ir.com/completed'
	encoded = base64.b64encode (bytes(object_name, "utf-8"))
	r = requests.post(link,data=encoded,headers={'x-api-key':token})
	return json.loads(r.text)
	
def encrypt(gzipfile,password,salt):
	f = createkey(password,salt)
	with io.FileIO(gzipfile,'rb') as r:
		data = r.read()
	r.close()
	encrypted = f.encrypt(data)
	with io.FileIO(gzipfile+'.encrypted','wb') as w:
		w.write(encrypted)
	w.close()
	os.remove(gzipfile)
	return gzipfile+'.encrypted'

def uploadfile(gzipfile,url,token,region,snapshot):
	object_name = region+'|'+snapshot+'|'+gzipfile+'.encrypted'
	encoded = base64.b64encode (bytes(object_name, "utf-8"))
	r = requests.post(url,data=encoded,headers={'x-api-key':token})
	uploadlink = json.loads(r.text)
	return uploadlink
	
def compress(fname):
	with io.FileIO(fname, 'rb') as r:
		with gzip.open(fname+'.gz','wb') as w:
			for b in r:
				w.write(b)
		w.close()
	r.close()
	os.remove(fname)
	return fname+'.gz'

def image(region,snapshot,count,token):
	status = 'START'
	link = 'https://upload.'+region+'.4n6ir.com'
	client = boto3.client('ebs', region_name=region)
	while(status):
		if status == 'START':
			response = client.list_snapshot_blocks(SnapshotId=snapshot)
			for block in response['Blocks']:
				download = client.get_snapshot_block(SnapshotId=snapshot,
													 BlockIndex=block['BlockIndex'],
													 BlockToken=block['BlockToken'])
				sha256_hash = hashlib.sha256()
				with io.FileIO(snapshot+'.tmp', 'wb') as f:
					for b in download['BlockData']:
						sha256_hash.update(b)
						f.write(b)
				f.close()
				fname = str(block['BlockIndex'])+'_'+snapshot+'_'+sha256_hash.hexdigest()+'_'+str(response['VolumeSize'])+'_'+str(response['BlockSize'])+'_'+str(count)
				shutil.move(snapshot+'.tmp',fname)
				gzipfile = compress(fname)
				uploadlink = uploadfile(gzipfile,link,token,region,snapshot)
				encrypted = encrypt(gzipfile,uploadlink['pwd'],uploadlink['salt'])
				with open(encrypted, 'rb') as f:
					files = {'file': (encrypted, f)}
					http_response = requests.post(uploadlink['url'], data=uploadlink['fields'], files=files)
				f.close()
				os.remove(encrypted)
			try:
				status = response['NextToken']
			except:
				status = ''
				continue
		else:
			response = client.list_snapshot_blocks(SnapshotId=snapshot,NextToken=status)
			for block in response['Blocks']:
				download = client.get_snapshot_block(SnapshotId=snapshot,
													 BlockIndex=block['BlockIndex'],
													 BlockToken=block['BlockToken'])
				sha256_hash = hashlib.sha256()
				with io.FileIO(snapshot+'.tmp', 'wb') as f:
					for b in download['BlockData']:
						sha256_hash.update(b)
						f.write(b)
				f.close()
				fname = str(block['BlockIndex'])+'_'+snapshot+'_'+sha256_hash.hexdigest()+'_'+str(response['VolumeSize'])+'_'+str(response['BlockSize'])+'_'+str(count)
				shutil.move(snapshot+'.tmp',fname)	
				gzipfile = compress(fname)
				uploadlink = uploadfile(gzipfile,url,token,region,snapshot)
				encrypted = encrypt(gzipfile,uploadlink['pwd'],uploadlink['salt'])
				with open(encrypted, 'rb') as f:
					files = {'file': (encrypted, f)}
					http_response = requests.post(uploadlink['url'], data=uploadlink['fields'], files=files)
				f.close()
				os.remove(encrypted)
			try:
				status = response['NextToken']
			except:
				status = ''
				continue

def blocks(region,snapshot,token):
	count = 0
	status = 'START'
	f = open('report_'+snapshot+'.txt','w')
	client = boto3.client('ebs', region_name=region)
	while(status):
		if status == 'START':
			response = client.list_snapshot_blocks(SnapshotId=snapshot)
			for block in response['Blocks']:
				f.write(str(block['BlockIndex'])+'\n')
				count = count + 1
			try:
				status = response['NextToken']
			except:
				status = ''
				continue
		else:
			response = client.list_snapshot_blocks(SnapshotId=snapshot,NextToken=status)
			for block in response['Blocks']:
				f.write(str(block['BlockIndex'])+'\n')
				count = count + 1
			try:
				status = response['NextToken']
			except:
				status = ''
				continue
	f.close()
	
	gzipfile = compress('report_'+snapshot+'.txt')
	link = 'https://upload.'+region+'.4n6ir.com'
	uploadlink = uploadfile(gzipfile,link,token,region,snapshot)
	encrypted = encrypt(gzipfile,uploadlink['pwd'],uploadlink['salt'])
	
	with open(encrypted, 'rb') as f:
		files = {'file': (encrypted, f)}
		http_response = requests.post(uploadlink['url'], data=uploadlink['fields'], files=files)
	f.close()
	os.remove(encrypted)
	
	return count

def main():
	parser = argparse.ArgumentParser(description="Snapshot 4n6ir Imager for Docker v0.1.1")
	required = parser.add_argument_group('Required')
	required.add_argument("--region", type=str, help="us-east-2", required=True)
	required.add_argument("--snapshot", type=str, help="snap-056e0b1bd07ad91b2", required=True)
	required.add_argument("--token", type=str, help="abacadaba-abacadaba-abacadaba", required=True)
	args = parser.parse_args()
	
	count = blocks(args.region,args.snapshot,args.token)
	
	print('Snapshot 4n6ir Imager for Docker v0.1.1\n')
	print('\tRegion: \t'+args.region)
	print('\tSnapshot: \t'+args.snapshot)
	print('\tBlocks: \t'+str(count))
	
	image(args.region,args.snapshot,count,args.token)
	status = completed(args.region,args.snapshot,args.token,count)	

	print('\tCompleted: \t'+status)
	
if __name__ == "__main__":
	main()