Merge pull request #58 from prio-data/update_show_tables

upgrades and fixes to documentation

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "viewser"
-version = "6.2.0"
+version = "6.3.0"
 description = "The Views 3 CLI tool"
 authors = ["peder2911 <pglandsverk@gmail.com>"]
 readme = "README.md"

viewser/cli.py

@@ -41,8 +41,9 @@ def viewser(ctx: click.Context, debug: bool, json: bool):
 
 viewser.add_command(queryset.cli)
 viewser.add_command(config.cli)
-viewser.add_command(documentation.tables_cli)
+viewser.add_command(documentation.features_cli)
 viewser.add_command(documentation.transforms_cli)
+viewser.add_command(documentation.transform_cli)
 viewser.add_command(notebooks.cli)
 viewser.add_command(system.cli)
 viewser.add_command(help.cli)

viewser/commands/documentation/__init__.py

@@ -1 +1 @@
-from .cli import tables_cli, transforms_cli
+from .cli import features_cli, transforms_cli, transform_cli

viewser/commands/documentation/cli.py

@@ -4,91 +4,42 @@
 import click
 import pandas as pd
 import json
+import requests
 from views_schema import docs as schema
 from viewser import settings
 from . import formatting, operations
 
-@click.group(name = "tables", short_help = "show information about available tables")
+
+@click.group(name="features", short_help="show information about available features")
 @click.pass_obj
-def tables_cli(obj: Dict[str, Any]):
+def features_cli(obj: Dict[str, Any]):
     """
-    Commands used to inspect available tables and columns.
+    Commands used to inspect available features.
     """
-    obj["operations"]       = operations.DocumentationCrudOperations(settings.config_get("REMOTE_URL"), "tables")
+#    obj["operations"] = operations.DocumentationCrudOperations(settings.config_get("REMOTE_URL"), "features")
     obj["detail_formatter"] = formatting.DocumentationDetailFormatter()
-    obj["list_formatter"]   = formatting.DocumentationTableFormatter()
+    obj["list_formatter"] = formatting.DocumentationTableFormatter()
+
 
-@tables_cli.command(name="list", short_help="show available tables")
+@features_cli.command(name="list", short_help="show available features at specified loa")
+@click.argument("loa")
 @click.pass_obj
-def list_tables(obj: Dict[str, Any]):
+def list_features(obj: Dict[str, Any], loa: str):
     """
-    list_tables
+    list_features
     ===========
 
-    Show all available tables.
+    Show all available features at specified loa.
     """
 
+    obj["operations"] = operations.DocumentationCrudOperations(settings.config_get("REMOTE_URL"),
+                                                               f"features/{loa}/")
     response_dict = json.loads(obj["operations"].list())['entries']
-    response_df = pd.DataFrame.from_dict(response_dict)['name'].sort_values().reset_index(drop=True).to_string()
+    response_df = (pd.DataFrame.from_dict(response_dict).drop(columns=['entries', 'data']).reset_index(drop=True).
+                   to_string())
 
     click.echo(response_df)
 
-@tables_cli.command(name="show", short_help="inspect table or column")
-@click.argument("table-name")
-@click.argument("column-name", required=False)
-@click.pass_obj
-def show_tables(
-        obj: Dict[str, Any],
-        table_name: str,
-        column_name: Optional[str] = None):
-    """
-    Show either a table or a column, depending on whether one argument is
-    passed (table name), or two arguments are passed (table name - column
-    name).
-    """
-    path = table_name
-
-    if column_name:
-        path += "/"+column_name
-        formatter = obj["detail_formatter"]
-    else:
-        formatter = obj["list_formatter"]
-
-    response_dict = json.loads(obj["operations"].show(path))['entries']
-
-    response_df = pd.DataFrame.from_dict(response_dict)['name'].sort_values().reset_index(drop=True).to_string()
-
-    click.echo(response_df)
-
-@tables_cli.command(name="annotate", short_help="add documentation text")
-@click.argument("content-file", type=click.File("r"))
-@click.argument("table-name")
-@click.argument("column-name", required=False)
-@click.option("--overwrite", is_flag = True, help="overwrite existing documentation")
-@click.pass_obj
-def annotate_table(
-        obj: Dict[str, Any],
-        content_file: io.BufferedReader,
-        table_name: str,
-        column_name: str,
-        overwrite: bool):
-    """
-    Annotate a table or a column. To annotate a table, pass a single argument
-    following the file containing the annotation. To annotate a column, pass
-    two arguments.
-    """
-
-    path = table_name
-    if column_name:
-        path += "/"+column_name
-        formatter = obj["detail_formatter"]
-    else:
-        formatter = obj["list_formatter"]
-
-    to_post = schema.PostedDocumentationPage(content = content_file.read())
-
-    click.echo(obj["operations"].post(to_post, path, overwrite = overwrite)
-        .either(obj["error_dumper"].formatted, formatter.formatted))
 
 @click.group(name="transforms")
 @click.pass_obj
@@ -101,51 +52,62 @@ def transforms_cli(obj: Dict[str, Any]):
     obj["table_formatter"] = formatting.DocumentationTableFormatter()
     obj["detail_formatter"] = formatting.FunctionDetailFormatter()
 
+
 @transforms_cli.command(name="list", short_help="show all available transforms")
 @click.pass_obj
 def list_transforms(obj: Dict[str, Any]):
     """
     List all available transforms.
     """
-    response_dict = json.loads(obj["operations"].list())['entries']
-    response_df = pd.DataFrame.from_dict(response_dict)['path'].sort_values().reset_index(drop=True).to_string()
+
+    response = requests.get(url=f'{settings.REMOTE_URL}/transforms')
+
+    response_df = pd.read_parquet(io.BytesIO(response.content))
+
     click.echo(response_df)
 
-@transforms_cli.command(name="show",short_help="show details about a transform")
-@click.argument("transform-name", type=str)
+
+@transforms_cli.command(name="at_loa", short_help="show transforms at a given loa")
+@click.argument("loa", type=str)
 @click.pass_obj
 def show_transform(
         obj: Dict[str, Any],
-        transform_name: str):
+        loa: str):
     """
     Show details about a transform, such as which arguments it takes, and
     which level of analysis it is applicable to.
     """
 
-    response_dict = json.loads(obj["operations"].show(transform_name))
+    response = requests.get(f'{settings.REMOTE_URL}/transforms/{loa}')
+
+    response_df = pd.read_parquet(io.BytesIO(response.content))
 
-    if len(response_dict['entries'])>0:
-        response_df = pd.DataFrame.from_dict(response_dict['entries'])['path'].sort_values().reset_index(drop=True).to_string()
-        click.echo(response_df)
-    else:
-        click.echo(response_dict['data']['docstring'])
+    click.echo(response_df)
 
 
-@transforms_cli.command(name="annotate", short_help="add documentation text")
-@click.argument("content-file", type=click.File("r"))
-@click.argument("transform-name")
-@click.option("--overwrite", is_flag = True, help="overwrite existing documentation")
+@click.group(name="transform")
 @click.pass_obj
-def annotate_transform(
-        obj: Dict[str, Any],
-        content_file: io.BufferedReader,
-        transform_name: str,
-        overwrite: bool = False):
+def transform_cli(obj: Dict[str, Any]):
+    """
+    Commands used to inspect available transforms.
+    """
+
+    obj["operations"] = operations.DocumentationCrudOperations(settings.config_get("REMOTE_URL"), "transform")
+    obj["table_formatter"] = formatting.DocumentationTableFormatter()
+    obj["detail_formatter"] = formatting.FunctionDetailFormatter()
+
+
+@transform_cli.command(name="show", short_help="show transform detail")
+@click.argument("transform_name", type=str)
+@click.pass_obj
+def transform(obj: Dict[str, Any], transform_name: str):
     """
-    Annotate a table or a column. To annotate a table, pass a single argument
-    following the file containing the annotation. To annotate a column, pass
-    two arguments.
+    List all available transforms.
     """
-    to_post = schema.PostedDocumentationPage(content = content_file.read())
-    click.echo(obj["operations"].post(to_post, transform_name, overwrite=overwrite)
-        .either(obj["error_dumper"].formatted, obj["detail_formatter"].formatted))
+
+    response = requests.get(url=f'{settings.REMOTE_URL}/transform/{transform_name}')
+
+    lines = response.content.decode().strip('"').split('\\n')
+
+    for line in lines:
+        print(line)

viewser/commands/documentation/operations.py

@@ -14,6 +14,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class DocumentationCrudOperations():
     """
     DocumentationCrudOperations
@@ -26,11 +27,11 @@ class DocumentationCrudOperations():
     Class for doing CRUD on documentation for views3
     """
 
-
     def __init__(self, base_url: str, path: str):
         self._base_url = os.path.join(base_url, path)
 
-    def post(self, posted: schema.PostedDocumentationPage, name: str = None, overwrite: bool = False)-> Either[Dump, schema.ViewsDoc]:
+    def post(self, posted: schema.PostedDocumentationPage, name: str = None, overwrite: bool = False) -> (
+            Either)[Dump, schema.ViewsDoc]:
 
         if name is None:
             name = posted.name
@@ -41,18 +42,17 @@ def post(self, posted: schema.PostedDocumentationPage, name: str = None, overwri
             except AssertionError:
                 return Left(errors.exists_error(name))
 
-        return (remotes.request(self._base_url,"POST", remotes.status_checks, name, data = Just(posted.dict()))
-            .then(lambda _: remotes.request(self._base_url, "GET", remotes.status_checks, name))
-            .then(self._deserialize))
+        return (remotes.request(self._base_url,"POST", remotes.status_checks, name, data=Just(posted.dict()))
+                .then(lambda _: remotes.request(self._base_url, "GET", remotes.status_checks, name))
+                .then(self._deserialize))
 
     def list(self) -> Either[Dump, schema.ViewsDoc]:
         return self.show("")
 
     def show(self, name: str) -> Either[Dump, schema.ViewsDoc]:
-        return remotes.request(self._base_url, "GET", remotes.status_checks, name).value.content.decode()#.then(self._deserialize)
-
+        return remotes.request(self._base_url, "GET", remotes.status_checks, name).value.content.decode()
 
-    def _exists(self,name: str) -> Either[Dump, bool]:
+    def _exists(self, name: str) -> Either[Dump, bool]:
         response = remotes.request(self._base_url, "GET", [], name)
         return response.is_right() and response.value.status_code == 200
 

viewser/remotes.py

@@ -46,9 +46,6 @@ def update_kwargs(kwargs, data):
         .maybe(request_kwargs, curry(update_kwargs, request_kwargs))
         )
 
-    print(request_args)
-    print(request_kwargs)
-
     try:
         response = requests.request(*request_args,**request_kwargs)
     except requests.exceptions.ConnectionError:
@@ -192,7 +189,6 @@ def request(
     data.then(str).then(lambda r: f"POSTing {r}").then(logger.debug)
 
     url = make_url(base_url, path, parameters)
-    print('url:',url)
     response = url.then(curry(make_request, method = method, json_data = data))
     return compose_checks(checks)(response)