// MigrateMyDatabase (migratemydatabase.eu): structure of this MongoDB database, for the free migration scan. // READ-ONLY. It runs only read commands (hello, buildInfo, dbStats, listCollections, listIndexes, $collStats and, // optionally, $sample). It never inserts, updates, deletes or creates anything, and it never outputs document data: // only collection settings, counts, sizes and index definitions (field names and directions). // // How to run (mongosh 1.x/2.x; MongoDB 4.4 or newer, Atlas, self-hosted, replica set or sharded cluster): // mongosh "mongodb+srv://user@cluster.example.net/mydb" --quiet --file mmdb-mongodb.js > mmdb-result.txt // mongosh "mongodb://localhost:27017/mydb" --quiet --file mmdb-mongodb.js > mmdb-result.txt // Then drop mmdb-result.txt on the page (or paste its contents). Use the database of your application in the URI. // // Optional sample (OFF by default): add --eval "var sample=200" BEFORE --file: // mongosh "mongodb://localhost:27017/mydb" --quiet --eval "var sample=200" --file mmdb-mongodb.js > mmdb-result.txt // It reads up to 200 random documents per collection ($sample) in mongosh on YOUR machine and outputs only field // NAMES (nested up to 3 levels, arrays as name[]), their BSON types, how many of the sampled documents have them // (percent) and the largest size seen in bytes. Values are never printed. // // Permissions: the built-in role "read" on the database is enough for everything except: serverStatus // (clusterMonitor; the storage engine is then taken from the collection statistics) and, on a sharded cluster before // MongoDB 8.0 ($listClusterCatalog), the shard key of sharded collections (read on the config database, or // clusterMonitor). A command that is not allowed leaves its fields empty and is listed in a "feature|access_denied" // line; the scan continues. // // Output: lines starting with MMDB1| (format: docs/COLLECTOR-FORMAT.md, kinds meta, db, ntable, nindex, nattr, // feature, end). Structure only, never data. (function () { 'use strict'; const VERSION = '1'; const MAX_DEPTH = 3; const SAMPLE = (typeof sample !== 'undefined' && Number(sample) > 0) ? Math.min(Math.floor(Number(sample)), 10000) : 0; const lines = []; const denied = []; const errors = []; const esc = (v) => (v === null || v === undefined) ? '' : (typeof v === 'boolean' ? (v ? '1' : '0') : String(v)) .replace(/\|/g, '/').replace(/[\r\n]/g, ' '); const out = (kind, ...f) => lines.push('MMDB1|' + kind + f.map((x) => '|' + esc(x)).join('')); const num = (x) => (x === null || x === undefined) ? undefined : Number(x.valueOf ? x.valueOf() : x); const mb = (n) => { n = num(n); if (n === undefined || isNaN(n)) return ''; return String(Number((n / 1048576).toFixed(3))); }; const flag = (b) => (b ? '1' : '0'); // run fn; on an authorization error remember the command as denied, on another error as api_error // (assign before return: mongosh awaits shell calls only in an assignment, so `return fn()` would let errors escape) function tryCall(name, fn) { try { const r = fn(); return r; } catch (e) { const msg = String((e && (e.codeName || e.message)) || e); const code = e && e.code; const list = (code === 13 || /Unauthorized|not authorized|requires authentication/i.test(msg)) ? denied : errors; if (!list.includes(name)) list.push(name); return undefined; } } out('meta', 'mongodb', VERSION, new Date().toISOString().slice(0, 19)); // ---------- database / server ---------- const dbName = db.getName(); const version = tryCall('buildInfo', () => db.version()) || ''; const bi = tryCall('buildInfo', () => db.runCommand({ buildInfo: 1 })) || {}; const env = bi.buildEnvironment || {}; const edition = (bi.modules || []).includes('enterprise') ? 'Enterprise' : 'Community'; const versionText = ['MongoDB', version, edition, env.target_os, env.target_arch].filter((x) => x).join(' '); const hello = tryCall('hello', () => db.runCommand({ hello: 1 })) || tryCall('isMaster', () => db.runCommand({ isMaster: 1 })) || {}; const topology = hello.msg === 'isdbgrid' ? 'sharded' : (hello.setName ? 'replicaset' : (hello.ok ? 'standalone' : '')); const ss = tryCall('serverStatus', () => db.adminCommand({ serverStatus: 1, repl: 0, metrics: 0, locks: 0 })) || {}; let engine = (ss.storageEngine && ss.storageEngine.name) || ''; // mongos has none: see the collections below const stats = tryCall('dbStats', () => db.runCommand({ dbStats: 1, scale: 1 })) || {}; // ---------- collections ---------- let infos = tryCall('listCollections', () => db.getCollectionInfos({}, { authorizedCollections: true })); if (!infos) { infos = tryCall('listCollections', () => db.runCommand({ listCollections: 1, nameOnly: true, authorizedCollections: true }).cursor.firstBatch); } infos = (infos || []).filter((c) => !c.name.startsWith('system.')).sort((a, b) => (a.name < b.name ? -1 : 1)); const names = new Set(infos.map((c) => c.name)); // sharded cluster: shard keys from $listClusterCatalog (MongoDB 8.0+, role read is enough); older versions // (unknown stage, not reported) fall back per collection to config.collections let catalog; if (topology === 'sharded') { try { const r = db.aggregate([{ $listClusterCatalog: {} }]).toArray(); catalog = new Map(r.map((x) => [String(x.ns).slice(dbName.length + 1), x])); } catch (e) { catalog = undefined; } } const feats = {}; const feat = (k, n, extra) => { const f = feats[k] || (feats[k] = { n: 0, extra: [] }); f.n += (n === undefined ? 1 : n); if (extra !== undefined) f.extra.push(extra); }; const ntables = []; const nindexes = []; const nattrs = []; let collCount = 0; const views = infos.filter((c) => c.type === 'view'); if (views.length) feat('view', views.length); for (const c of infos) { if (c.type === 'view') continue; collCount++; const name = c.name; const opt = c.options || {}; const coll = db.getCollection(name); const ts = c.type === 'timeseries' || !!opt.timeseries; if (opt.capped) feat('capped_collection'); if (ts) feat('timeseries_collection'); if (opt.validator && Object.keys(opt.validator).length) feat('validator'); if (opt.clusteredIndex) feat('clustered_collection'); if (opt.encryptedFields) feat('queryable_encryption'); if (opt.collation) feat('collation'); if (opt.changeStreamPreAndPostImages && opt.changeStreamPreAndPostImages.enabled) feat('change_stream_pre_post_images'); let count = tryCall('estimatedDocumentCount', () => coll.estimatedDocumentCount()); let st = tryCall('collStats', () => coll.aggregate([{ $collStats: { storageStats: {} } }]).toArray()); let size, storage, idxSize, sharded = false; if (st && st.length) { // one result per shard on a sharded cluster: add them up. Orphan documents (left on a shard after a chunk // migration until the range deleter removes them) are counted by the shard and by estimatedDocumentCount: // subtract them, and their share of the data size (storage and index sizes are physical: left as they are) let orphans = 0, shardCount = 0, haveShardCount = false; for (const s of st) { const x = s.storageStats || {}; const n = num(x.count) || 0; const o = Math.min(num(x.numOrphanDocs) || 0, n); orphans += o; size = (size || 0) + (num(x.size) || 0) * (n ? (n - o) / n : 1); storage = (storage || 0) + (num(x.storageSize) || 0); idxSize = (idxSize || 0) + (num(x.totalIndexSize) || 0); if (x.count !== undefined) { haveShardCount = true; shardCount += n - o; } if (x.sharded === true) sharded = true; if (!engine && x.wiredTiger) engine = 'wiredTiger'; if (!engine && x.inMemory) engine = 'inMemory'; } if (count === undefined && haveShardCount) count = shardCount; else if (count !== undefined && orphans) count = Math.max(num(count) - orphans, 0); if (st.length > 1) sharded = true; } let shardKey = ''; if (topology === 'sharded') { let key; if (catalog) { const x = catalog.get(name); if (x && x.sharded) key = x.shardKey || {}; } else { const cfg = tryCall('config.collections', () => db.getSiblingDB('config').collections.findOne({ _id: dbName + '.' + name })); if (cfg && cfg.key && !cfg.dropped) key = cfg.key; else if (!cfg && denied.includes('config.collections') && !sharded) { // no access to config: the (deprecated) collStats command still says whether it is sharded, not the key try { const r = db.runCommand({ collStats: name, scale: 1 }); if (r && r.sharded === true) sharded = true; } catch (e) { /* ignore */ } } } if (key) { sharded = true; shardKey = Object.entries(key).map(([k, v]) => k + ':' + v).join(','); } } if (sharded) feat('sharded_collection'); // indexes let ttlField = ''; const idx = tryCall('listIndexes', () => coll.getIndexes()) || []; for (const ix of idx) { const key = ix.key || {}; const vals = Object.values(key).map(String); let keys; if (vals.includes('text')) { const textFields = Object.keys(ix.weights || {}).map((k) => k + ':text'); keys = Object.entries(key).filter(([k]) => k !== '_fts' && k !== '_ftsx').map(([k, v]) => k + ':' + v) .concat(textFields); } else { keys = Object.entries(key).map(([k, v]) => k + ':' + v); } const ttl = ix.expireAfterSeconds !== undefined; const unique = !!ix.unique || ix.name === '_id_'; let kind; if (vals.includes('text')) kind = 'text'; else if (vals.includes('2dsphere')) kind = '2dsphere'; else if (vals.includes('2d')) kind = '2d'; else if (vals.includes('geoHaystack')) kind = 'geoHaystack'; else if (vals.includes('hashed')) kind = 'hashed'; else if (Object.keys(key).some((k) => k.endsWith('$**'))) kind = 'wildcard'; else if (ttl) kind = 'ttl'; else if (ix.unique) kind = 'unique'; else if (Object.keys(key).length > 1) kind = 'compound'; else kind = 'single'; if (ttl && !ts) ttlField = ttlField || Object.keys(key)[0]; if (['text', '2dsphere', '2d', 'geoHaystack'].includes(kind)) feat(kind === 'text' ? 'text_index' : 'geo_index'); if (kind === 'hashed') feat('hashed_index'); if (kind === 'wildcard') feat('wildcard_index'); if (ttl) feat('ttl_index'); if (ix.unique) feat('unique_index'); if (ix.partialFilterExpression) feat('partial_index'); nindexes.push([name, ix.name, kind, keys.join(','), '', flag(unique), // appended (MongoDB): sparse, partial, TTL seconds, hidden, number of key fields, own collation, clustered flag(ix.sparse), flag(ix.partialFilterExpression), ttl ? num(ix.expireAfterSeconds) : '', flag(ix.hidden), keys.length, flag(ix.collation), flag(ix.clustered)]); } if (ts && opt.timeseries && opt.expireAfterSeconds !== undefined) ttlField = opt.timeseries.timeField; // optional local sample: field names, types and sizes only let sampled = '', maxDoc = ''; if (SAMPLE > 0) { const docs = tryCall('$sample', () => coll.aggregate([{ $sample: { size: SAMPLE } }], { promoteValues: false, promoteLongs: false }).toArray()); if (docs) { const paths = new Map(); let mx = 0; for (const d of docs) { mx = Math.max(mx, bsonSize(d)); const seen = new Map(); for (const [k, v] of Object.entries(d)) walk(k, v, 1, seen); for (const [p, s] of seen) { const e = paths.get(p) || { t: new Set(), n: 0, max: 0 }; s.t.forEach((t) => e.t.add(t)); e.n++; e.max = Math.max(e.max, s.max); paths.set(p, e); } } sampled = docs.length; maxDoc = docs.length ? mx : ''; for (const p of [...paths.keys()].sort()) { const e = paths.get(p); nattrs.push([name, p, [...e.t].sort().join(','), docs.length ? Math.round(100 * e.n / docs.length) : 0, e.max]); } } } ntables.push([name, num(count), mb(size), shardKey, '', opt.capped ? 'capped' : '', '', '', '', ttlField, flag(opt.validator && Object.keys(opt.validator).length), opt.encryptedFields ? 'QE' : '', '', '', '', flag(ts), // appended: sampled documents, largest sampled document (bytes), storage size MB, index size MB sampled, maxDoc, mb(storage), mb(idxSize)]); } // GridFS buckets: .files together with .chunks const buckets = [...names].filter((n) => n.endsWith('.files') && names.has(n.slice(0, -6) + '.chunks')).map((n) => n.slice(0, -6)); if (buckets.length) feat('gridfs', buckets.length, buckets.join(',')); out('db', dbName, versionText, version, topology, engine, mb(stats.dataSize), mb(stats.indexSize), // appended: storage size MB, number of collections (without views), number of views mb(stats.storageSize), collCount, views.length); ntables.forEach((f) => out('ntable', ...f)); nindexes.forEach((f) => out('nindex', ...f)); nattrs.forEach((f) => out('nattr', ...f)); Object.keys(feats).sort().forEach((k) => out('feature', k, feats[k].n, feats[k].extra.join(','))); out('feature', 'change_streams_unknown', 0, ''); out('feature', 'transactions_unknown', 0, ''); if (denied.length) out('feature', 'access_denied', denied.length, denied.join(',')); if (errors.length) out('feature', 'api_error', errors.length, errors.join(',')); out('end', lines.length); lines.forEach((l) => print(l)); // ---------- helpers for the sample (types and sizes, never values) ---------- function typeOf(v) { if (v === null) return 'null'; if (v === undefined) return 'undefined'; if (typeof v === 'string') return 'string'; if (typeof v === 'boolean') return 'bool'; if (typeof v === 'number') return Number.isInteger(v) ? 'int' : 'double'; if (typeof v === 'bigint') return 'long'; if (Array.isArray(v)) return 'array'; const tag = Object.prototype.toString.call(v); // instanceof fails across the shell's realms if (tag === '[object Date]') return 'date'; if (tag === '[object RegExp]') return 'regex'; switch (v._bsontype) { case 'ObjectId': case 'ObjectID': return 'objectId'; case 'Int32': return 'int'; case 'Double': return 'double'; case 'Long': return 'long'; case 'Decimal128': return 'decimal'; case 'Binary': return 'binData'; case 'Timestamp': return 'timestamp'; case 'BSONRegExp': return 'regex'; case 'Code': return v.scope ? 'javascriptWithScope' : 'javascript'; case 'BSONSymbol': case 'Symbol': return 'symbol'; case 'DBRef': return 'dbRef'; case 'MinKey': return 'minKey'; case 'MaxKey': return 'maxKey'; case undefined: return 'object'; default: return String(v._bsontype); } } function bsonSize(d) { try { return bsonsize(d); } catch (e) { try { return Object.bsonsize(d); } catch (e2) { return 0; } } } // size of one value as stored in BSON (approximately: element without its name) function valueSize(v) { const n = bsonSize({ '': v }); return n > 7 ? n - 7 : n; // 4 length + 1 type + 1 empty name + 1 terminator } function walk(path, v, depth, seen) { const t = typeOf(v); const e = seen.get(path) || { t: new Set(), max: 0 }; e.t.add(t); e.max = Math.max(e.max, valueSize(v)); seen.set(path, e); if (t === 'object' && depth < MAX_DEPTH) { for (const [k, c] of Object.entries(v)) walk(path + '.' + k, c, depth + 1, seen); } else if (t === 'array') { for (const c of v) walk(path + '[]', c, depth, seen); } } })();