Extended JS interface -> Markdown generator (#4275)

* Extended Markdown generator

* Fix linter errors

* Review fixes

js/.gitignore

@@ -1,6 +1,7 @@
 node_modules
 npm-debug.log
 build
+docs
 .build
 .coverage
 .dist

js/package.json

@@ -29,6 +29,8 @@
     "build:app": "webpack --config webpack/app",
     "build:lib": "webpack --config webpack/libraries",
     "build:dll": "webpack --config webpack/vendor",
+    "build:markdown": "babel-node ./scripts/build-rpc-markdown.js",
+    "build:json": "babel-node ./scripts/build-rpc-json.js",
     "ci:build": "npm run ci:build:lib && npm run ci:build:dll && npm run ci:build:app",
     "ci:build:app": "NODE_ENV=production webpack --config webpack/app",
     "ci:build:lib": "NODE_ENV=production webpack --config webpack/libraries",
@@ -74,6 +76,7 @@
     "chai": "3.5.0",
     "chai-as-promised": "6.0.0",
     "chai-enzyme": "0.6.1",
+    "chalk": "1.1.3",
     "circular-dependency-plugin": "2.0.0",
     "copy-webpack-plugin": "4.0.1",
     "core-js": "2.4.1",

js/scripts/build-rpc-json.js

@@ -17,9 +17,9 @@
 import fs from 'fs';
 import path from 'path';
 
-import interfaces from '../';
+import interfaces from '../src/jsonrpc';
 
-const INDEX_JSON = path.join(__dirname, '../../release/index.json');
+const INDEX_JSON = path.join(__dirname, '../release/index.json');
 const methods = [];
 
 function formatDescription (obj) {

js/scripts/build-rpc-markdown.js

@@ -0,0 +1,328 @@
+// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+import fs from 'fs';
+import path from 'path';
+import chalk from 'chalk';
+
+import { DUMMY } from '../src/jsonrpc/helpers';
+import { BlockNumber } from '../src/jsonrpc/types';
+import interfaces from '../src/jsonrpc';
+
+const ROOT_DIR = path.join(__dirname, '../docs');
+
+if (!fs.existsSync(ROOT_DIR)) {
+  fs.mkdirSync(ROOT_DIR);
+}
+
+const type2print = new WeakMap();
+
+type2print.set(BlockNumber, 'Quantity|Tag');
+
+// INFO Logging helper
+function info (log) {
+  console.log(chalk.blue(`INFO:\t${log}`));
+}
+
+// WARN Logging helper
+function warn (log) {
+  console.warn(chalk.yellow(`WARN:\t${log}`));
+}
+
+// ERROR Logging helper
+function error (log) {
+  console.error(chalk.red(`ERROR:\t${log}`));
+}
+
+function printType (type) {
+  return type2print.get(type) || type.name;
+}
+
+function formatDescription (obj, prefix = '', indent = '') {
+  const optional = obj.optional ? '(optional) ' : '';
+  const defaults = obj.default ? `(default: \`${obj.default}\`) ` : '';
+
+  return `${indent}${prefix}\`${printType(obj.type)}\` - ${optional}${defaults}${obj.desc}`;
+}
+
+function formatType (obj) {
+  if (obj == null) {
+    return obj;
+  }
+
+  if (obj.type === Object && obj.details) {
+    const sub = Object.keys(obj.details).map((key) => {
+      return formatDescription(obj.details[key], `\`${key}\`: `, '    - ');
+    }).join('\n');
+
+    return `${formatDescription(obj)}\n${sub}`;
+  } else if (obj.type && obj.type.name) {
+    return formatDescription(obj);
+  }
+
+  return obj;
+}
+
+const rpcReqTemplate = {
+  method: 'web3_clientVersion',
+  params: [],
+  id: 1,
+  jsonrpc: '2.0'
+};
+
+// Checks if the value passed in is a DUMMY object placeholder for `{ ... }``
+function isDummy (val) {
+  return val === DUMMY;
+}
+
+const { isArray } = Array;
+
+// Checks if the value passed is a plain old JS object
+function isObject (val) {
+  return val != null && val.constructor === Object;
+}
+
+// Checks if a field definition has an example,
+// or describes an object with fields that recursively have examples of their own,
+// or is optional.
+function hasExample ({ optional, example, details } = {}) {
+  if (optional || example !== undefined) {
+    return true;
+  }
+
+  if (details !== undefined) {
+    const values = Object.keys(details).map((key) => details[key]);
+
+    return values.every(hasExample);
+  }
+
+  return false;
+}
+
+// Remove all optional (trailing) params without examples from an array
+function removeOptionalWithoutExamples (arr) {
+  return arr.filter(({ optional, example, details }) => {
+    return !optional || example !== undefined || details !== undefined;
+  });
+}
+
+// Grabs JSON compatible
+function getExample (obj) {
+  if (isArray(obj)) {
+    return removeOptionalWithoutExamples(obj).map(getExample);
+  }
+
+  const { example, details } = obj;
+
+  if (example === undefined && details !== undefined) {
+    const nested = {};
+
+    Object.keys(details).forEach((key) => {
+      nested[key] = getExample(details[key]);
+    });
+
+    return nested;
+  }
+
+  return example;
+}
+
+function stringifyExample (example, dent = '') {
+  const indent = `${dent}  `;
+
+  if (example === DUMMY) {
+    return '{ ... }';
+  }
+
+  if (isArray(example)) {
+    const last = example.length - 1;
+
+    // If all elements are dummies, print out a single line.
+    // Also covers empty arrays.
+    if (example.every(isDummy)) {
+      const dummies = example.map(_ => '{ ... }');
+
+      return `[${dummies.join(', ')}]`;
+    }
+
+    // For arrays containing just one object or string, don't unwind the array to multiline
+    if (last === 0 && (isObject(example[0]) || typeof example[0] === 'string')) {
+      return `[${stringifyExample(example[0], dent)}]`;
+    }
+
+    const elements = example.map((value, index) => {
+      const comma = index !== last ? ',' : '';
+      const comment = value != null && value._comment ? ` // ${value._comment}` : '';
+
+      return `${stringifyExample(value, indent)}${comma}${comment}`;
+    });
+
+    return `[\n${indent}${elements.join(`\n${indent}`)}\n${dent}]`;
+  }
+
+  if (isObject(example)) {
+    const keys = Object.keys(example);
+    const last = keys.length - 1;
+
+    // print out an empty object
+    if (last === -1) {
+      return '{}';
+    }
+
+    const elements = keys.map((key, index) => {
+      const value = example[key];
+      const comma = index !== last ? ',' : '';
+      const comment = value && value._comment ? ` // ${example[key]._comment}` : '';
+
+      return `${JSON.stringify(key)}: ${stringifyExample(value, indent)}${comma}${comment}`;
+    });
+
+    return `{\n${indent}${elements.join(`\n${indent}`)}\n${dent}}`;
+  }
+
+  return JSON.stringify(example); // .replace(/"\$DUMMY\$"/g, '{ ... }');
+}
+
+function buildExample (name, method) {
+  // deprecated, don't care
+  if (method.deprecated) {
+    return '';
+  }
+
+  const logPostfix = method.subdoc ? ` (${method.subdoc})` : '';
+
+  const hasReqExample = method.params.every(hasExample);
+  const hasResExample = hasExample(method.returns);
+
+  if (!hasReqExample && !hasResExample) {
+    error(`${name} has no examples${logPostfix}`);
+
+    return '';
+  }
+
+  const examples = [];
+
+  if (hasReqExample) {
+    const params = getExample(method.params);
+    const req = JSON.stringify(Object.assign({}, rpcReqTemplate, { method: name, params })).replace(/"\$DUMMY\$"/g, '{ ... }');
+
+    examples.push(`Request\n\`\`\`bash\ncurl --data '${req}' -H "Content-Type: application/json" -X POST localhost:8545\n\`\`\``);
+  } else {
+    warn(`${name} has a response example but not a request example${logPostfix}`);
+  }
+
+  if (hasResExample) {
+    const res = stringifyExample({
+      id: 1,
+      jsonrpc: '2.0',
+      result: getExample(method.returns)
+    });
+
+    examples.push(`Response\n\`\`\`js\n${res}\n\`\`\``);
+  } else {
+    if (typeof method.returns === 'string') {
+      info(`${name} has a request example and only text description for response${logPostfix}`);
+    } else {
+      warn(`${name} has a request example but not a response example${logPostfix}`);
+    }
+  }
+
+  return `\n\n#### Example\n\n${examples.join('\n\n')}`;
+}
+
+function buildParameters (params) {
+  if (params.length === 0) {
+    return '';
+  }
+
+  let md = `0. ${params.map(formatType).join('\n0. ')}`;
+
+  if (params.length > 0 && params.every(hasExample) && params[0].example !== DUMMY) {
+    const example = getExample(params);
+
+    md = `${md}\n\n\`\`\`js\nparams: ${stringifyExample(example)}\n\`\`\``;
+  }
+
+  return md;
+}
+
+Object.keys(interfaces).sort().forEach((group) => {
+  const spec = interfaces[group];
+
+  for (const key in spec) {
+    const method = spec[key];
+
+    if (!method || !method.subdoc) {
+      continue;
+    }
+
+    const subgroup = `${group}_${method.subdoc}`;
+
+    interfaces[subgroup] = interfaces[subgroup] || {};
+
+    interfaces[subgroup][key] = method;
+    delete spec[key];
+  }
+});
+
+Object.keys(interfaces).sort().forEach((group) => {
+  let preamble = `# The \`${group}\` Module`;
+  let markdown = `## JSON-RPC methods\n`;
+
+  const spec = interfaces[group];
+
+  if (spec._preamble) {
+    preamble = `${preamble}\n\n${spec._preamble}`;
+  }
+
+  const content = [];
+  const tocMain = [];
+  const tocSections = {};
+
+  Object.keys(spec).sort().forEach((iname) => {
+    const method = spec[iname];
+    const name = `${group.replace(/_.*$/, '')}_${iname}`;
+
+    if (method.nodoc || method.deprecated) {
+      info(`Skipping ${name}: ${method.nodoc || 'Deprecated'}`);
+
+      return;
+    }
+
+    const desc = method.desc;
+    const params = buildParameters(method.params);
+    const returns = `- ${formatType(method.returns)}`;
+    const example = buildExample(name, method);
+
+    const { section } = method;
+    const toc = section ? tocSections[section] = tocSections[section] || [] : tocMain;
+
+    toc.push(`- [${name}](#${name.toLowerCase()})`);
+    content.push(`### ${name}\n\n${desc}\n\n#### Parameters\n\n${params || 'None'}\n\n#### Returns\n\n${returns || 'None'}${example}`);
+  });
+
+  markdown = `${markdown}\n${tocMain.join('\n')}`;
+
+  Object.keys(tocSections).sort().forEach((section) => {
+    markdown = `${markdown}\n\n#### ${section}\n${tocSections[section].join('\n')}`;
+  });
+
+  markdown = `${markdown}\n\n## JSON-RPC API Reference\n\n${content.join('\n\n***\n\n')}\n\n`;
+
+  const mdFile = path.join(ROOT_DIR, `${group}.md`);
+
+  fs.writeFileSync(mdFile, `${preamble}\n\n${markdown}`, 'utf8');
+});

js/src/jsonrpc/README.md

@@ -12,9 +12,10 @@ JSON file of all ethereum's rpc methods supported by parity
 0. Branch
 0. Add the missing interfaces only into `src/interfaces/*.js`
 0. Parameters (array) & Returns take objects of type
-    - `{ type: [Array|Boolean|Object|String|...], desc: 'some description' }`
+    - `{ type: [Array|Boolean|Object|String|...], desc: 'some description', example: 100|'0xff'|{ ... } }`
     - Types are built-in JS types or those defined in `src/types.js` (e.g. `BlockNumber`, `Quantity`, etc.)
     - If a formatter is required, add it as `format: 'string-type'`
-0. Run the lint & tests, `npm run lint && npm run testOnce`
-0. Generate via `npm run build` which outputs `index.js`, `index.json` & `interfaces.md` (Only required until Travis is fully in-place)
-0. Check-in and make a PR
+0. Run the lint & tests, `npm run lint && npm run test`
+0. Generate via `npm run build` which outputs `index.js` & `index.json`.
+0. (optional) Generate docs via `npm run build:markdown` which outputs `md` files to `./docs`.
+0. Check-in and make a PR.

js/src/jsonrpc/generator/build-markdown.js

@@ -1,69 +0,0 @@
-// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
-// This file is part of Parity.
-
-// Parity is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Parity is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
-
-import fs from 'fs';
-import path from 'path';
-
-import interfaces from '../';
-
-const MARKDOWN = path.join(__dirname, '../../release/interfaces.md');
-
-let preamble = '# interfaces\n';
-let markdown = '';
-
-function formatDescription (obj, prefix = '', indent = '') {
-  const optional = obj.optional ? '(optional) ' : '';
-  const defaults = obj.default ? `(default: ${obj.default}) ` : '';
-
-  return `${indent}- ${prefix}\`${obj.type.name}\` - ${optional}${defaults}${obj.desc}`;
-}
-
-function formatType (obj) {
-  if (obj.type === Object && obj.details) {
-    const sub = Object.keys(obj.details).sort().map((key) => {
-      return formatDescription(obj.details[key], `\`${key}\`/`, '    ');
-    }).join('\n');
-
-    return `${formatDescription(obj)}\n${sub}`;
-  } else if (obj.type && obj.type.name) {
-    return formatDescription(obj);
-  }
-
-  return obj;
-}
-
-Object.keys(interfaces).sort().forEach((group) => {
-  let content = '';
-
-  preamble = `${preamble}\n- [${group}](#${group})`;
-  markdown = `${markdown}\n## ${group}\n`;
-
-  Object.keys(interfaces[group]).sort().forEach((iname) => {
-    const method = interfaces[group][iname];
-    const name = `${group}_${iname}`;
-    const deprecated = method.deprecated ? ' (Deprecated and not supported, to be removed in a future version)' : '';
-    const desc = `${method.desc}${deprecated}`;
-    const params = method.params.map(formatType).join('\n');
-    const returns = formatType(method.returns);
-
-    markdown = `${markdown}\n- [${name}](#${name})`;
-    content = `${content}### ${name}\n\n${desc}\n\n#### parameters\n\n${params || 'none'}\n\n#### returns\n\n${returns || 'none'}\n\n`;
-  });
-
-  markdown = `${markdown}\n\n${content}`;
-});
-
-fs.writeFileSync(MARKDOWN, `${preamble}\n\n${markdown}`, 'utf8');

js/src/jsonrpc/helpers.js

@@ -0,0 +1,62 @@
+// Copyright 2015, 2016 Parity Technologies (UK) Ltd.
+// This file is part of Parity.
+
+// Parity is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Parity is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Parity.  If not, see <http://www.gnu.org/licenses/>.
+
+// Placeholders for objects with undefined fields, will show up in docs as `{ ... }`
+export const DUMMY = '$DUMMY$';
+
+// Enrich the API spec by additional markdown-formatted preamble
+export function withPreamble (preamble, spec) {
+  Object.defineProperty(spec, '_preamble', {
+    value: preamble.trim(),
+    enumerable: false
+  });
+
+  return spec;
+}
+
+// Enrich any example value with a comment to print in the docs
+export function withComment (example, comment) {
+  const constructor = example == null ? null : example.constructor;
+
+  if (constructor === Object || constructor === Array) {
+    Object.defineProperty(example, '_comment', {
+      value: comment,
+      enumerable: false
+    });
+
+    return example;
+  }
+
+  // Convert primitives
+  return new PrimitiveWithComment(example, comment);
+}
+
+// Turn a decimal number into a hexadecimal string with comment to it's original value
+export function fromDecimal (decimal) {
+  return withComment(`0x${decimal.toString(16)}`, decimal.toString());
+}
+
+// Internal helper
+class PrimitiveWithComment {
+  constructor (primitive, comment) {
+    this._value = primitive;
+    this._comment = comment;
+  }
+
+  toJSON () {
+    return this._value;
+  }
+}