added aria2 tool and croissant

cm-mlops/script/get-aria2/README-extra.md

@@ -0,0 +1,9 @@
+# Some commands
+
+```bash
+cmr "get aria2" --version=1.37.0
+cmr "get aria2" --install
+cmr "get aria2" --path={path to the directory with aria2}
+cmr "get aria2" --input={full path to aria2}
+cmr "get aria2" --shell
+```

cm-mlops/script/get-aria2/_cm.yaml

@@ -0,0 +1,37 @@
+alias: get-aria2
+uid: d83419a90a0c40d0
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+cache: true
+
+category: Detection or installation of tools and artifacts
+
+input_mapping:
+  install: CM_FORCE_INSTALL
+  src: CM_ARIA2_BUILD_FROM_SRC
+
+deps:
+  - tags: detect,cpu
+  - tags: detect,os
+
+#called after preprocess from customize.py
+#prehook_deps:
+#  - tags: print,native,hello-world
+
+env:
+  CM_REQUIRE_INSTALL: no
+  CM_ARIA2_DEFAULT_INSTALL_VERSION: "1.37.0"
+
+new_env_keys:
+  - CM_ARIA2_*
+  - +PATH
+
+print_env_at_the_end:
+  CM_ARIA2_INSTALLED_PATH: Path to the tool
+
+tags:
+- get
+- aria2
+- get-aria2

cm-mlops/script/get-aria2/customize.py

@@ -0,0 +1,122 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    # Pre-set by CM
+    os_info = i['os_info']
+    env = i['env']
+    recursion_spaces = i['recursion_spaces']
+    automation = i['automation']
+    run_script_input = i['run_script_input']
+
+    # Check if a given tool is already installed
+    file_name_core = 'aria2c'
+    file_name = file_name_core+'.exe' if os_info['platform'] == 'windows' else file_name_core
+
+    force_install = env.get('CM_FORCE_INSTALL', False) == True
+
+    if not force_install:
+        r = i['automation'].find_artifact({'file_name': file_name,
+                                           'env':env,
+                                           'os_info':os_info,
+                                           'default_path_env_key': 'PATH',
+                                           'detect_version':True,
+                                           'env_path_key':'CM_ARIA2_BIN_WITH_PATH',
+                                           'run_script_input':i['run_script_input'],
+                                           'recursion_spaces':recursion_spaces})
+        if r['return'] >0 : 
+            if r['return'] == 16:
+                # Not found, try install
+                force_install = True
+            else:
+                return r
+
+    # Force install
+    if force_install:
+        # Attempt to run installer
+        version = env.get('CM_VERSION','')
+        if version == '': version = env['CM_ARIA2_DEFAULT_INSTALL_VERSION']
+
+        if os_info['platform'] == 'windows':
+            archive = 'aria2-{}-win-64bit-build1'
+            ext = '.zip'
+            ext2 = ''
+        else:
+            archive = 'aria2-{}' 
+            ext = '.tar.bz2'
+            ext2 = '.tar'
+
+        archive = archive.format(version)
+        archive_with_ext = archive+ext
+
+        env['CM_ARIA2_DOWNLOAD_DIR'] = archive
+
+        env['CM_ARIA2_DOWNLOAD_FILE'] = archive_with_ext
+        if ext2!='':
+            env['CM_ARIA2_DOWNLOAD_FILE2'] = archive+ext2
+
+        url = 'https://github.com/aria2/aria2/releases/download/release-{}/{}'.format(version, archive_with_ext)
+        env['CM_ARIA2_DOWNLOAD_URL'] = url
+
+        print ('URL to download ARIA2: {}'.format(url))
+
+        r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'})
+        if r['return']>0: return r
+
+        if os_info['platform'] == 'windows' or env.get('CM_ARIA2_BUILD_FROM_SRC', '').lower() == 'true':
+            install_path = os.path.join(os.getcwd(), archive)
+
+            path_to_file = os.path.join(install_path, file_name)
+            if not os.path.isfile(path_to_file):
+                return {'return':1, 'error':'file not found: {}'.format(path_to_file)}
+
+            env['CM_ARIA2_BIN_WITH_PATH'] = path_to_file
+            env['CM_ARIA2_INSTALLED_TO_CACHE'] = 'yes'
+        else:
+            path_to_bin = r['env_tmp'].get('CM_ARIA2_BIN_WITH_PATH','')
+            env['CM_ARIA2_BIN_WITH_PATH'] = path_to_bin
+
+            r = i['automation'].find_artifact({'file_name': file_name,
+                                               'env':env,
+                                               'os_info':os_info,
+                                               'default_path_env_key': 'PATH',
+                                               'detect_version':True,
+                                               'env_path_key':'CM_ARIA2_BIN_WITH_PATH',
+                                               'run_script_input':i['run_script_input'],
+                                               'recursion_spaces':recursion_spaces})
+            if r['return']>0: return r
+
+    return {'return':0}
+
+def detect_version(i):
+    env = i['env']
+
+    r = i['automation'].parse_version({'match_text': r'aria2 version\s*([\d.]+)',
+                                       'group_number': 1,
+                                       'env_key':'CM_ARIA2_VERSION',
+                                       'which_env':i['env']})
+    if r['return'] >0: return r
+
+    version = r['version']
+    print (i['recursion_spaces'] + '    Detected version: {}'.format(version))
+
+    return {'return':0, 'version':version}
+
+def postprocess(i):
+
+    env = i['env']
+    r = detect_version(i)
+    if r['return'] >0: return r
+
+    version = r['version']
+    found_file_path = env['CM_ARIA2_BIN_WITH_PATH']
+
+    found_path = os.path.dirname(found_file_path)
+
+    env['CM_ARIA2_INSTALLED_PATH'] = found_path
+
+    if env.get('CM_ARIA2_INSTALLED_TO_CACHE','')=='yes':
+        env['+PATH'] = [env['CM_ARIA2_INSTALLED_PATH']]
+
+    return {'return':0, 'version': version}

cm-mlops/script/get-aria2/install.bat

@@ -0,0 +1,9 @@
+echo.
+
+del /Q /S %CM_ARIA2_DOWNLOAD_FILE%
+
+wget --no-check-certificate %CM_ARIA2_DOWNLOAD_URL%
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
+
+unzip -o -q %CM_ARIA2_DOWNLOAD_FILE%
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%

cm-mlops/script/get-aria2/install.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+echo ""
+
+if [[ "${CM_ARIA2_BUILD_FROM_SRC}" == "True" ]]; then
+
+  echo "Building from sources ..."
+  echo ""
+
+  rm -rf ${CM_ARIA2_DOWNLOAD_FILE}
+  rm -rf ${CM_ARIA2_DOWNLOAD_FILE2}
+
+  wget --no-check-certificate ${CM_ARIA2_DOWNLOAD_URL}
+  test $? -eq 0 || exit $?
+
+  bzip2 -d ${CM_ARIA2_DOWNLOAD_FILE}
+  test $? -eq 0 || exit $?
+
+  tar xvf ${CM_ARIA2_DOWNLOAD_FILE2}
+  test $? -eq 0 || exit $?
+
+  cd ${CM_ARIA2_DOWNLOAD_DIR}
+  test $? -eq 0 || exit $?
+
+  ./configure --prefix=$PWD/bin
+  test $? -eq 0 || exit $?
+
+  make
+  test $? -eq 0 || exit $?
+
+  make install
+  test $? -eq 0 || exit $?
+
+else
+  echo "Installing binary via sudo ..."
+  echo ""
+
+  cmd="sudo ${CM_HOST_OS_PACKAGE_MANAGER} install aria2"
+  echo "$cmd"
+
+  $cmd
+  test $? -eq 0 || exit $?
+
+  path_to_bin=`which aria2c`
+  echo "CM_ARIA2_BIN_WITH_PATH=$path_to_bin" > tmp-run-env.out
+
+fi

cm-mlops/script/get-aria2/run.bat

@@ -0,0 +1,4 @@
+rem Detect version
+
+%CM_ARIA2_BIN_WITH_PATH% --version > tmp-ver.out
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%

cm-mlops/script/get-aria2/run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Detect version
+
+${CM_ARIA2_BIN_WITH_PATH} --version > tmp-ver.out
+test $? -eq 0 || exit 1

cm-mlops/script/get-croissant/README.md

@@ -0,0 +1,128 @@
+Automatically generated README for this automation recipe: **get-croissant**
+
+Category: **AI/ML datasets**
+
+License: **Apache 2.0**
+
+Maintainers: [Public MLCommons Task Force on Automation and Reproducibility](https://github.com/mlcommons/ck/blob/master/docs/taskforce.md)
+
+---
+*[ [Online info and GUI to run this CM script](https://access.cknowledge.org/playground/?action=scripts&name=get-croissant,8fd653eac8da4c14) ]*
+
+---
+#### Summary
+
+* CM GitHub repository: *[mlcommons@ck](https://github.com/mlcommons/ck/tree/dev/cm-mlops)*
+* GitHub directory for this script: *[GitHub](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant)*
+* CM meta description for this script: *[_cm.json](_cm.json)*
+* All CM tags to find and reuse this script (see in above meta description): *get,mlcommons,croissant*
+* Output cached? *True*
+* See [pipeline of dependencies](#dependencies-on-other-cm-scripts) on other CM scripts
+
+
+---
+### Reuse this script in your project
+
+#### Install MLCommons CM automation meta-framework
+
+* [Install CM](https://access.cknowledge.org/playground/?action=install)
+* [CM Getting Started Guide](https://github.com/mlcommons/ck/blob/master/docs/getting-started.md)
+
+#### Pull CM repository with this automation recipe (CM script)
+
+```cm pull repo mlcommons@ck```
+
+#### Print CM help from the command line
+
+````cmr "get mlcommons croissant" --help````
+
+#### Customize and run this script from the command line with different variations and flags
+
+`cm run script --tags=get,mlcommons,croissant`
+
+`cm run script --tags=get,mlcommons,croissant `
+
+*or*
+
+`cmr "get mlcommons croissant"`
+
+`cmr "get mlcommons croissant " `
+
+
+#### Run this script from Python
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+```python
+
+import cmind
+
+r = cmind.access({'action':'run'
+                  'automation':'script',
+                  'tags':'get,mlcommons,croissant'
+                  'out':'con',
+                  ...
+                  (other input keys for this script)
+                  ...
+                 })
+
+if r['return']>0:
+    print (r['error'])
+
+```
+
+</details>
+
+
+#### Run this script via GUI
+
+```cmr "cm gui" --script="get,mlcommons,croissant"```
+
+Use this [online GUI](https://cKnowledge.org/cm-gui/?tags=get,mlcommons,croissant) to generate CM CMD.
+
+#### Run this script via Docker (beta)
+
+`cm docker script "get mlcommons croissant" `
+
+___
+### Customization
+
+#### Default environment
+
+<details>
+<summary>Click here to expand this section.</summary>
+
+These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.json` or using script flags.
+
+
+</details>
+
+___
+### Dependencies on other CM scripts
+
+
+  1. ***Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json)***
+     * detect,os
+       - CM script: [detect-os](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/detect-os)
+     * get,python3
+       * CM names: `--adr.['python3', 'python']...`
+       - CM script: [get-python3](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-python3)
+     * git,repo,_repo.https://github.com/mlcommons/croissant
+       * CM names: `--adr.['git-mlcommons-croissant']...`
+       - CM script: [get-git-repo](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-git-repo)
+  1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/customize.py)***
+  1. Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json)
+  1. ***Run native script if exists***
+     * [run.bat](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/run.bat)
+     * [run.sh](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/run.sh)
+  1. Read "posthook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json)
+  1. ***Run "postrocess" function from [customize.py](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/customize.py)***
+  1. Read "post_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/dev/cm-mlops/script/get-croissant/_cm.json)
+
+___
+### Script output
+`cmr "get mlcommons croissant "  -j`
+#### New environment keys (filter)
+
+#### New environment keys auto-detected from customize