Merge pull request #4 from dylibso/upgrade-schema-definition

Upgrade to new v1 schema def

README.md

@@ -1,9 +1,74 @@
 # xtp-bindgen
 
-This is a typescript library for parsing an [XTP schema document](https://docs.xtp.dylibso.com/docs/host-usage/xtp-schema).
-This is experimental, more coming soon.
+XTP Bindgen is an open source framework to generate PDK bindings for [Extism](https://extism.org) plug-ins.
+It's used by the [XTP Platform](https://www.getxtp.com/), but can be used outside of the platform to define
+any Extism compatible plug-in system.
+
+## How Does It Work?
+
+[Extism](https://extism.org) has a very simple bytes-in / bytes-out interface. The host and the guest must agree
+on the interface used (exports functions, imports functions, and types). The purpose of this project is
+to create a canonical document and set of tools for defining this interface and generating bindings. That
+document is the [XTP Schema](https://docs.xtp.dylibso.com/docs/concepts/xtp-schema). This is an IDL of our creation.
+It is similar to [OpenAPI](https://www.openapis.org/), but is focused on defining plug-in interfaces, not HTTP interfaces.
+
+Once you define your interface as a schema, you can use one of the bindgens to generate code. We have some official
+bindgens available for writing PDKs, but more will be availble soon for a variety of purposes. There may also be community
+bindgens you can use.
+
+* [TypeScript](https://github.com/dylibso/xtp-typescript-bindgen)
+* [Go](https://github.com/dylibso/xtp-go-bindgen)
+* [C#](https://github.com/dylibso/xtp-csharp-bindgen)
+
+## How Do I Use A Bindgen?
+
+You can use the [XTP CLI](https://docs.xtp.dylibso.com/docs/cli/) to generate plug-ins.
+
+> *Note*: You don't need to authenticate to XTP to use the plugin generator
+
+Use the `plugin init` command to generate a plugin:
 
 ```
-npm install @dylibso/xtp-bindgen
+xtp plugin init \
+    --schema myschema.yaml \
+    --template @dylibso/xtp-typescript-bindgen \
+    --path ./myplugin \
+    --feature-flags none
 ```
 
+You can point to a bindgen template on github or directly to a bindgen bundle.
+
+
+## How Do I Write A Bindgen?
+
+
+A bindgen is simply a zip file with the following attributes:
+
+* `plugin.wasm` an extism plugin to generate the code
+* `config.yaml` a config file for the generator
+* `template` a template folder of files and templates that the generator will recursively process
+
+For reference, Here is what is inside the typescript bindgen:
+
+```
+$ tree bundle
+bundle
+├── config.yaml
+├── plugin.wasm
+└── template
+    ├── esbuild.js
+    ├── package.json.ejs
+    ├── src
+    │   ├── index.d.ts.ejs
+    │   ├── index.ts.ejs
+    │   ├── main.ts.ejs
+    │   └── pdk.ts.ejs
+    ├── tsconfig.json
+    └── xtp.toml.ejs
+```
+
+The XTP CLI will download and unpack this template, it will load the plugin, and it will recursively walk
+through the template and pass each file through the plugin to be rendered. Our official bindgens
+currently use typescript and EJS to render the projects, but these are not mandatory.
+It can be any Extism plug-in.
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dylibso/xtp-bindgen",
-  "version": "0.0.12",
+  "version": "1.0.0-rc.5",
   "description": "XTP bindgen helper library",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

src/index.ts

@@ -5,6 +5,7 @@ import {
   isProperty,
   parseAndNormalizeJson,
   Property,
+  Parameter,
   XtpSchema,
 } from "./normalizer";
 import { CodeSample } from "./parser";
@@ -54,7 +55,7 @@ function codeSamples(ex: Export, lang: string): CodeSample[] {
 }
 
 // template helpers
-function hasComment(p: Property | Export | Import | null | undefined): boolean {
+function hasComment(p: Parameter | Property | Export | Import | null | undefined): boolean {
   if (!p) return false;
 
   if (isProperty(p)) {
@@ -83,25 +84,26 @@ function formatCommentBlock(s: string | null, prefix?: string) {
   return s.trimEnd().replace(/\n/g, `\n${prefix}`);
 }
 
-
-function isDateTime(p: Property | null): boolean {
-  if (!p) return false
-  return p.type === 'string' && p.format === 'date-time'
-}
-
-function isJsonEncoded(p: Property | null): boolean {
+function isJsonEncoded(p: Parameter | null): boolean {
   if (!p) return false
   return p.contentType === 'application/json'
 }
 
-function isUtf8Encoded(p: Property | null): boolean {
+function isUtf8Encoded(p: Parameter | null): boolean {
   if (!p) return false
-  return p.contentType === 'text/plain; charset=UTF-8'
+  return p.contentType === 'text/plain; charset=utf-8'
 }
 
-function isPrimitive(p: Property): boolean {
+function isPrimitive(p: Property | Parameter): boolean {
   if (!p.$ref) return true
-  return !!p.$ref.enum && !p.$ref.properties
+  // enums are currently primitive (strings)
+  // schemas with props are not (needs to be encoded)
+  return !!p.$ref.enum || !p.$ref.properties
+}
+
+function isDateTime(p: Property | Parameter | null): boolean {
+  if (!p) return false
+  return p.type === 'string' && p.format === 'date-time'
 }
 
 export const helpers = {

src/normalizer.ts

@@ -8,6 +8,7 @@ export interface Property extends Omit<parser.Property, '$ref'> {
   '$ref': Schema | null;
   nullable: boolean;
   items?: XtpItemType;
+  name: string;
 }
 
 export function isProperty(p: any): p is Property {
@@ -16,6 +17,7 @@ export function isProperty(p: any): p is Property {
 
 export interface Schema extends Omit<parser.Schema, 'properties'> {
   properties: Property[];
+  name: string;
 }
 
 export type SchemaMap = {
@@ -34,13 +36,14 @@ export type Version = 'v0' | 'v1';
 export type XtpType = parser.XtpType
 export type XtpFormat = parser.XtpFormat
 export type MimeType = parser.MimeType
+export type Parameter = parser.Parameter
 
 export interface Export {
   name: string;
   description?: string;
   codeSamples?: parser.CodeSample[];
-  input?: Property;
-  output?: Property;
+  input?: Parameter;
+  output?: Parameter;
 }
 
 export function isExport(e: any): e is Export {
@@ -81,15 +84,25 @@ function normalizeV0Schema(parsed: parser.V0Schema): XtpSchema {
 function parseSchemaRef(ref: string): string {
   const parts = ref.split('/')
   if (parts[0] !== '#') throw Error("Not a valid ref " + ref)
-  if (parts[1] !== 'schemas') throw Error("Not a valid ref " + ref)
-  return parts[2]
+  if (parts[1] !== 'components') throw Error("Not a valid ref " + ref)
+  if (parts[2] !== 'schemas') throw Error("Not a valid ref " + ref)
+  return parts[3]
 }
 
-function normalizeProp(p: Property | XtpItemType, s: Schema) {
+function normalizeProp(p: Parameter | Property | XtpItemType, s: Schema) {
   p.$ref = s
-  p.type = s.type || 'string' // TODO: revisit string default, isn't type required?
-  p.contentType = p.contentType || s.contentType
   p.description = p.description || s.description
+  // double ensure that content types are lowercase
+  if ('contentType' in p) {
+    p.contentType = p.contentType.toLowerCase() as MimeType
+  }
+  if (!p.type) p.type = 'string'
+  if (s.type) {
+    // if it's not an object assume it's a string
+    if (s.type === 'object') {
+      p.type = 'object'
+    }
+  }
 }
 
 function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema {
@@ -99,42 +112,63 @@ function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema {
   const schemas: SchemaMap = {}
 
   // need to index all the schemas first
-  parsed.schemas?.forEach(s => {
-    schemas[s.name] = s as Schema
-  })
+  for (const name in parsed.components?.schemas) {
+    const s = parsed.components.schemas[name]
+    const properties: Property[] = []
+    for (const pName in s.properties) {
+      const p = s.properties[pName] as Property
+      p.name = pName
+      properties.push(p)
+    }
+
+    // overwrite the name
+    // overwrite new properties shape
+    schemas[name] = {
+      ...s,
+      name,
+      properties,
+    }
+  }
 
   // denormalize all the properties in a second loop
-  parsed.schemas?.forEach(s => {
+  for (const name in schemas) {
+    const s = schemas[name]
+
     s.properties?.forEach((p, idx) => {
       // link the property with a reference to the schema if it has a ref
-      if (p.$ref) {
+      // need to get the ref from the parsed (raw) property
+      const rawProp = parsed.components!.schemas![name].properties![p.name]
+
+      if (rawProp.$ref) {
         normalizeProp(
-          schemas[s.name].properties[idx],
-          schemas[parseSchemaRef(p.$ref)]
+          schemas[name].properties[idx],
+          schemas[parseSchemaRef(rawProp.$ref)]
         )
       }
 
-      if (p.items?.$ref) {
+      if (rawProp.items?.$ref) {
         normalizeProp(
           //@ts-ignore
           p.items!,
-          schemas[parseSchemaRef(p.items!.$ref)]
+          schemas[parseSchemaRef(rawProp.items!.$ref)]
         )
       }
 
-      // add set nullable property from the required array
-      // TODO: consider supporting nullable instead of required
-      // @ts-ignore
-      p.nullable = !s.required?.includes(p.name)
+      // coerce to false by default
+      p.nullable = p.nullable || false
     })
-  })
+  }
 
   // denormalize all the exports
-  parsed.exports.forEach(ex => {
+  for (const name in parsed.exports) {
+    let ex = parsed.exports[name]
+
     if (parser.isComplexExport(ex)) {
       // they have the same type
       // deref input and output
       const normEx = ex as Export
+      normEx.name = name
+
       if (ex.input?.$ref) {
         normalizeProp(
           normEx.input!,
@@ -166,16 +200,20 @@ function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema {
       exports.push(normEx)
     } else if (parser.isSimpleExport(ex)) {
       // it's just a name
-      exports.push({ name: ex })
+      exports.push({ name })
     } else {
       throw new NormalizerError("Unable to match export to a simple or a complex export")
     }
-  })
+  }
 
   // denormalize all the imports
-  parsed.imports?.forEach(im => {
+  for (const name in parsed.imports) {
+    const im = parsed.imports![name]
+
     // they have the same type
     const normIm = im as Import
+    normIm.name = name
+
     // deref input and output
     if (im.input?.$ref) {
       normalizeProp(
@@ -206,7 +244,7 @@ function normalizeV1Schema(parsed: parser.V1Schema): XtpSchema {
     }
 
     imports.push(normIm)
-  })
+  }
 
   return {
     version,