#!/usr/bin/env python3 # MigrateMyDatabase (migratemydatabase.eu): structure of your Amazon DynamoDB tables, for the free migration scan. # READ-ONLY. It calls only List/Describe APIs (and Scan when you ask for the optional sample). It never creates, # changes or deletes anything, and it never outputs item data: only table settings, keys, indexes and sizes. # # How to run (Python 3 with boto3; AWS CloudShell has both, nothing to install): # python3 mmdb-dynamodb.py --region eu-west-1 -o mmdb-result.txt # python3 mmdb-dynamodb.py --region eu-west-1 --profile prod --tables Orders,Customers # python3 mmdb-dynamodb.py --region eu-west-1 --sample 200 -o mmdb-result.txt # Then drop mmdb-result.txt on the page (or paste its contents). # # Options: # --region R AWS region to read (default: the region of your profile / AWS_REGION) # --profile P named profile from ~/.aws/config (optional) # --endpoint-url U other endpoint, e.g. http://localhost:8000 for DynamoDB Local (testing) # --tables A,B only these tables (comma separated); default: all tables in the region # --sample N OFF by default (0). Reads up to N items per table (one Scan page, eventually consistent) # on YOUR machine and outputs only attribute NAMES, their types (S, N, M, L ...), how many of # the sampled items have them (percent) and the largest size seen in bytes. Values never # leave your machine and are not written to the output. Costs about N/2 x 4 KB read units. # -o FILE write the result to FILE instead of the screen # # Minimal IAM policy (read-only): # {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Resource": "*", "Action": [ # "dynamodb:ListTables", "dynamodb:DescribeTable", "dynamodb:DescribeTimeToLive", # "dynamodb:DescribeContinuousBackups", # "dynamodb:Scan", <- only needed for --sample # "sts:GetCallerIdentity"]}]} <- optional (account id; it is normally read from the table ARN) # The AWS managed policy AmazonDynamoDBReadOnlyAccess also works. A call that is denied leaves its fields empty # and is listed in a "feature|access_denied" line; the rest of the scan continues. Other API failures (for # example DescribeContinuousBackups, which DynamoDB Local does not implement) go to a "feature|api_error" line. # With --endpoint-url, STS is not called; the account id comes from the table ARN. # # Output: lines starting with MMDB1| (format: docs/COLLECTOR-FORMAT.md, kinds meta, db, ntable, nindex, nattr, # feature, end). Structure only, never data. import argparse import base64 import datetime import sys try: import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError except ImportError: sys.exit("boto3 is needed: run this in AWS CloudShell, or: python3 -m pip install boto3") VERSION = "1" MAX_DEPTH = 3 DENIED_CODES = {"AccessDeniedException", "AccessDenied", "UnauthorizedOperation", "NotAuthorized"} def esc(v): if v is None: return "" if isinstance(v, bool): return "1" if v else "0" return str(v).replace("|", "/").replace("\r", " ").replace("\n", " ") def mb(n): if n is None: return "" s = "%.3f" % (n / 1048576.0) return s.rstrip("0").rstrip(".") if "." in s else s class Collector: def __init__(self, args): self.args = args self.lines = [] self.denied = [] # API names that were denied self.errors = [] # API names that failed otherwise (e.g. not supported by DynamoDB Local) self.last_code = "" cfg = Config(retries={"max_attempts": 4, "mode": "standard"}, connect_timeout=10, read_timeout=30) session = boto3.Session(profile_name=args.profile, region_name=args.region) if args.profile \ else boto3.Session(region_name=args.region) self.region = session.region_name or "" if not self.region: sys.exit("No region: pass --region (e.g. --region eu-west-1)") self.ddb = session.client("dynamodb", endpoint_url=args.endpoint_url, config=cfg) self.session = session self.cfg = cfg def out(self, kind, *fields): self.lines.append("MMDB1|" + kind + "".join("|" + esc(f) for f in fields)) def call(self, api, fn, **kw): """Run one API call; on AccessDenied / other error return None and remember the API name.""" self.last_code = "" try: return fn(**kw) except ClientError as e: code = e.response.get("Error", {}).get("Code", "") self.last_code = code if code == "ResourceNotFoundException": # table name from --tables that does not exist print("warning: %s: %s not found" % (api, kw.get("TableName", "")), file=sys.stderr) elif code in DENIED_CODES or "AccessDenied" in code: if api not in self.denied: self.denied.append(api) else: if api not in self.errors: self.errors.append(api) print("warning: %s failed: %s" % (api, code or e), file=sys.stderr) return None except BotoCoreError as e: if api not in self.errors: self.errors.append(api) print("warning: %s failed: %s" % (api, e), file=sys.stderr) return None def list_tables(self): names, start = [], None while True: kw = {"ExclusiveStartTableName": start} if start else {} r = self.call("ListTables", self.ddb.list_tables, **kw) if r is None: return None names.extend(r.get("TableNames", [])) start = r.get("LastEvaluatedTableName") if not start: return names def account_id(self, arns): for arn in arns: parts = (arn or "").split(":") if len(parts) > 4 and parts[4]: return parts[4] if self.args.endpoint_url: # test endpoint: do not call the real STS with test credentials return "" sts = self.session.client("sts", config=self.cfg) r = self.call("GetCallerIdentity", sts.get_caller_identity) return r.get("Account", "") if r else "" def run(self): a = self.args self.out("meta", "dynamodb", VERSION, datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")) listed = self.list_tables() if a.tables: names = [t.strip() for t in a.tables.split(",") if t.strip()] else: names = listed or [] tables, ntables, nindexes, nattrs = [], [], [], [] feats = {} def feat(k, n=1): feats[k] = feats.get(k, 0) + n for name in names: r = self.call("DescribeTable", self.ddb.describe_table, TableName=name) if r is None: if self.last_code != "ResourceNotFoundException": ntables.append([name]) # name known, details not readable continue t = r["Table"] tables.append(t) types = {d["AttributeName"]: d["AttributeType"] for d in t.get("AttributeDefinitions", [])} def keys(schema): return [k["AttributeName"] + ":" + types.get(k["AttributeName"], "") for k in schema] ks = t.get("KeySchema", []) pk = keys([k for k in ks if k["KeyType"] == "HASH"]) sk = keys([k for k in ks if k["KeyType"] == "RANGE"]) billing = (t.get("BillingModeSummary") or {}).get("BillingMode") or "PROVISIONED" pt = t.get("ProvisionedThroughput") or {} rcu = pt.get("ReadCapacityUnits") if billing == "PROVISIONED" else "" wcu = pt.get("WriteCapacityUnits") if billing == "PROVISIONED" else "" feat("provisioned_table" if billing == "PROVISIONED" else "on_demand_table") ss = t.get("StreamSpecification") or {} stream = ss.get("StreamViewType", "") if ss.get("StreamEnabled") else "" if stream: feat("stream") tclass = (t.get("TableClassSummary") or {}).get("TableClass") or "STANDARD" if tclass != "STANDARD": feat("infrequent_access_class") sse = t.get("SSEDescription") or {} enc = "AWS_OWNED" if sse.get("Status") in ("ENABLED", "ENABLING", "UPDATING"): enc = sse.get("SSEType") or "KMS" # KMS (AWS managed or customer key) / AES256 if enc == "KMS": feat("kms_encryption") replicas = [x.get("RegionName") for x in t.get("Replicas", []) if x.get("RegionName")] gregions = "" if replicas: gregions = ",".join(sorted(set(replicas + [self.region]))) feat("global_table") dprot = t.get("DeletionProtectionEnabled") dprot = "" if dprot is None else ("1" if dprot else "0") if dprot == "1": feat("deletion_protection") ttl_attr = "" r = self.call("DescribeTimeToLive", self.ddb.describe_time_to_live, TableName=name) if r is not None: d = r.get("TimeToLiveDescription") or {} if d.get("TimeToLiveStatus") in ("ENABLED", "ENABLING"): ttl_attr = d.get("AttributeName", "") feat("ttl") pitr = "" r = self.call("DescribeContinuousBackups", self.ddb.describe_continuous_backups, TableName=name) if r is not None: d = (r.get("ContinuousBackupsDescription") or {}).get("PointInTimeRecoveryDescription") or {} pitr = "1" if d.get("PointInTimeRecoveryStatus") == "ENABLED" else "0" if pitr == "1": feat("pitr") sampled, maxitem = "", "" if a.sample > 0: s = self.sample(name) if s is not None: n, paths, maxitem = s sampled = n for p in sorted(paths): ts, cnt, mx = paths[p] nattrs.append([name, p, ",".join(sorted(ts)), round(100.0 * cnt / n) if n else 0, mx]) ntables.append([name, t.get("ItemCount"), mb(t.get("TableSizeBytes")), ",".join(pk), ",".join(sk), billing, rcu, wcu, stream, ttl_attr, tclass, enc, gregions, pitr, dprot, "", sampled, maxitem]) for kind, lst in (("GSI", t.get("GlobalSecondaryIndexes", [])), ("LSI", t.get("LocalSecondaryIndexes", []))): for ix in lst: feat(kind.lower()) pr = ix.get("Projection") or {} ipt = ix.get("ProvisionedThroughput") or {} nindexes.append([name, ix.get("IndexName"), kind, ",".join(keys(ix.get("KeySchema", []))), pr.get("ProjectionType", ""), "0", ipt.get("ReadCapacityUnits", "") if kind == "GSI" and billing == "PROVISIONED" else "", ipt.get("WriteCapacityUnits", "") if kind == "GSI" and billing == "PROVISIONED" else "", ix.get("ItemCount", ""), mb(ix.get("IndexSizeBytes")), len(pr.get("NonKeyAttributes", [])) if pr.get("ProjectionType") == "INCLUDE" else ""]) account = self.account_id([t.get("TableArn") for t in tables]) self.out("db", self.region, account, len(listed) if listed is not None else (len(names) if a.tables else "")) for f in ntables: self.out("ntable", *f) for f in nindexes: self.out("nindex", *f) for f in nattrs: self.out("nattr", *f) for k in sorted(feats): self.out("feature", k, feats[k], "") self.out("feature", "transactions_unknown", 0, "") if self.denied: self.out("feature", "access_denied", len(self.denied), ",".join(self.denied)) if self.errors: self.out("feature", "api_error", len(self.errors), ",".join(self.errors)) self.out("end", len(self.lines)) return self.lines # ---------- optional local sample: attribute names, types and sizes only ---------- def sample(self, name): r = self.call("Scan", self.ddb.scan, TableName=name, Limit=self.args.sample, ConsistentRead=False) if r is None: return None items = r.get("Items", []) paths = {} # path -> [set(types), items having it, max size] maxitem = 0 for item in items: seen = {} total = 0 for k, v in item.items(): total += len(k.encode("utf-8")) + size_of(v) walk(k, v, 1, seen) maxitem = max(maxitem, total) for p, (ts, mx) in seen.items(): e = paths.setdefault(p, [set(), 0, 0]) e[0].update(ts) e[1] += 1 e[2] = max(e[2], mx) return len(items), paths, (maxitem if items else "") def type_of(v): return next(iter(v)) if isinstance(v, dict) and len(v) == 1 else "?" def size_of(v): """Approximate DynamoDB size in bytes of one attribute value (low-level JSON), per the AWS sizing rules.""" t = type_of(v) x = v.get(t) if isinstance(v, dict) else None if t == "S": return len(x.encode("utf-8")) if t == "N": return len(x.lstrip("-").replace(".", "").lstrip("0")) // 2 + 2 if t == "B": return len(x) if isinstance(x, (bytes, bytearray)) else len(base64.b64decode(x)) if t in ("BOOL", "NULL"): return 1 if t == "SS": return sum(len(s.encode("utf-8")) for s in x) if t == "NS": return sum(len(s.lstrip("-").replace(".", "").lstrip("0")) // 2 + 2 for s in x) if t == "BS": return sum(len(b) if isinstance(b, (bytes, bytearray)) else len(base64.b64decode(b)) for b in x) if t == "M": return 3 + sum(len(k.encode("utf-8")) + 1 + size_of(c) for k, c in x.items()) if t == "L": return 3 + sum(1 + size_of(c) for c in x) return 0 def walk(path, v, depth, seen): """Record type and size of path; descend into maps and lists up to MAX_DEPTH name levels.""" t = type_of(v) e = seen.setdefault(path, [set(), 0]) e[0].add(t) e[1] = max(e[1], size_of(v)) if t == "M" and depth < MAX_DEPTH: for k, c in v["M"].items(): walk(path + "." + k, c, depth + 1, seen) elif t == "L": for c in v["L"]: walk(path + "[]", c, depth, seen) def main(): p = argparse.ArgumentParser(description="MigrateMyDatabase read-only DynamoDB structure collector") p.add_argument("--region") p.add_argument("--profile") p.add_argument("--endpoint-url") p.add_argument("--tables", help="comma separated table names") p.add_argument("--sample", type=int, default=0, help="items per table to inspect locally (default 0 = off)") p.add_argument("-o", "--output") args = p.parse_args() if args.sample < 0 or args.sample > 10000: sys.exit("--sample must be between 0 and 10000") lines = Collector(args).run() text = "\n".join(lines) + "\n" if args.output: with open(args.output, "w", encoding="utf-8", newline="\n") as f: f.write(text) print("%d lines written to %s" % (len(lines), args.output), file=sys.stderr) else: sys.stdout.write(text) if __name__ == "__main__": main()