Create Schema Script

// @desc    Base names for types used by kdb+ Insights assembly files
typeNames:(.Q.t except" ")!key each(.Q.t except" ")$\:()
// @desc    Nested columns are simple plurals, except 'chars' which becomes 'string'
typeNames:@[;"C";:;`string]typeNames,{(upper key x)!`$(string value x),'"s"}typeNames
// @desc    Names for attributes used by assembly files
attrNames:`g`u`p`s!`grouped`unique`parted`sorted
// @desc    Wraps `meta`, stripping out virtual partition columns.
// @param   tbl     Table to be described
// @return  Table of metadata same as from `meta`
realMeta:{[tbl]m:meta tbl;if[1b~.Q.qp tbl;m:delete from m where c=.Q.pf];m}
// @desc    Given a table, produces a data structure representing the assembly configuration for its columns.
// @param   tbl     Table (as value, not name)
// @return  List of dicts of parameters for each column
describeCols:{[tbl] {if[`~x`type;x[`anymap]:`true];if[`=x`attrDisk;x:`attrDisk _x];x}each `name`type`attrDisk xcol `c`t`a#update typeNames t,attrNames a from 0!realMeta tbl}
// @desc    Gets name, storage type and value of each table loaded.
// @return  Table of name/type/value of tables
getTbls:{`name`type`val!/:{(x;((1b;0b;0)!`partitioned`splayed`basic) .Q.qp v;v:get x)}each tables[]}
// @desc    Combines output of `getTbls` and `describeCols` into an assembly configuration.
// @param   tblInfo Table output from `getTbls`
// @return  Dict of parameters for each table, including parameters for their columns
describeTbls:{[tblInfo] (tblInfo`name)!{if[`<>s:first x[`columns;;`name]where x[`columns;;`attrDisk]in`parted`sorted;x[`sortColsDisk]:enlist s];x} each`type`columns#update columns:describeCols each val from tblInfo}
// @desc    Out-of-band token that will be serialized into an invalid form to indicate where users must edit
EDITME:`$"@EDITME@"
// @desc    Takes the output of `describeTbls` and annotates it with necessary user edits.
// @param   dsc     Dict output from `describeTbls`
// @return  Pair `(dsc;msg)` of `dsc` with 'edit me' tokens added, and `msg` advising the user on the necessary edits
annotate:{[dsc]
 reminder:"\n";
 $[`json~args`fmt;
 [fn:"JSON";reminder:"# REMINDER: Remove these comments from the JSON file before using it, as JSON does not natively support comments.\n",reminder];
 fn:"YAML"];
 msg:"# NOTE: This script generates a preliminary ",fn," file for kdb+ insights schema.\n",
 "#\n",
 "# IMPORTANT: The generated file is NOT a valid kdb+ insights schema yet.\n",
 "# Manual review and updates are required for the following:\n",
 "# 1. Define the `prtnCol` for each partitioned table.\n",
 "# 2. Add necessary attributes for `idb` and `rdb` configurations.\n",
 "# 3. Review and update `sort columns` as needed, ensuring extra columns are added where appropriate.\n";
 dsc:{if[`partitioned=x`type;x[`prtnCol]:EDITME];x} each dsc;
 dsc:{if[99h<>type x;:x];$[`attrDisk in key x;x,`attrOrd`attrMem!(EDITME;EDITME);x]}@/:/:/:dsc;
 dsc:{if[`sortColsDisk in key x;x[`sortColsDisk],:EDITME];x}@/:dsc;
 if[@[{.Q.pf};`;`date]<>`date;msg,:"# 4. Convert database to be partitioned by date rather than ",(string .Q.pf),".\n"];
 if[count bad:where not `attrDisk in/:raze@'key@''dsc[p:where `partitioned=dsc[;`type];`columns];
 msg,:"# 5. Add attributes to tables: ",(", "sv string p bad),", or queries on them could be very slow.\n"];
 msg,:"#\n",
 "# Please address these points before using the schema.\n",
 reminder;
 (dsc;msg)
 }
// @desc    Indents every line in a string by two spaces.
// @param   x       Input string
// @return  Input string indented by two spaces
indent:{"\n"sv"  ",/:"\n"vs x}
// @desc    Serializes an atomic value. We use `.j.j` as JSON and YAML have compatible syntaxes for this. 'Edit me' tokens
//          are converted into the invalid unquoted string `@EDITME@` at this stage.
// @param   x       Atomic value
// @return  String representation of `x`
atomToStr:{$[x~EDITME;string EDITME;.j.j x]}
// @desc    Outputs JSON with block indentation (unlike .j.j) and with special treatment of 'edit me' tokens.
// @param   val     Value to be seralised
// @return  Serialised possibly-invalid JSON document
mkJSON:{[val]
 t:type val;$[
 t<0;atomToStr val;
 t within 1 19;"[",(", "sv .z.s each val),"]";
 t in 0 98h;"[\n",(indent",\n"sv .z.s each val),"\n]";
 t=99h;"{\n",(indent",\n"sv": "sv/:flip (atomToStr each key val;.z.s each value val)),"\n}"
 ]
 }
// @desc    Outputs YAML in the same format as used in assembly specifications, and with special treatment of 'edit me' tokens.
// @param   val     Value to be serialised
// @return  Serialised possibly-invalid YAML document
mkYAML:{[val]
 t:type val;$[
 t<0;atomToStr val;
 t within 1 19;"[",(", "sv .z.s each val),"]";
 t in 0 98h;"\n"sv{@[;0;:;"-"] indent x} each .z.s each val;
 t=99h;"\n"sv": "sv/:flip (string key val;{$["\n" in x;"\n",indent x;x]}each .z.s each value val)
 ]
 }
// @desc    Converts a dict of dicts into a list by turning each key into a `name` sub-key. YAML assembly files are structured as a dict with
//          table names as keys; JSON schemas in the UI and per-table YAML files in packages use a 'name' key for each table. This is used to
//          convert from the former to the latter structure.
// @param   dsc     Dict of dicts output by `annotate`
// @return  List of dicts 
restructure:{[dsc]
 ((enlist`name)!/:enlist each key dsc),'value dsc
 }
// @desc    Writes a dict of dicts from `annotate` to disk by writing each key to disk in a separate file corresponding to the table name.
// @param   dir     Directory to write files into
// @param   dsc     Dict of dicts output by `annotate`
// @param   msg     String output by `annotate`
pkgOut:{[dir;dsc;msg] {.Q.dd[x;` sv(z`name),`yaml]0:enlist y,mkYAML z}[dir;msg]each restructure dsc}
// @desc    Parses command-line arguments and calls functions to describe an output dictionary and output the description appropriately.
main:{
 args::.Q.opt .z.x;
 args::.Q.def[`db`out`fmt!(`.;`;`yaml)]args;
 args[`out]:$[null args`out;`;(args`out)like "/*";hsym args`out;.Q.dd[hsym`$system"cd";args`out]];
 @[system;"l ",string args`db;{[db;e]1"Cannot load database from ",db," ",e;exit 1}[string args`db]];
 v:annotate describeTbls getTbls[];
 dsc:v 0;
 msg:v 1;
 if[`package~args`fmt;pkgOut[args`out;dsc;msg];:()];
 $[`~args`out;-1;'[0:[args`out];enlist]] msg,((`json`yaml!('[mkJSON;restructure];mkYAML))args`fmt) dsc
 }
main[];exit 0