Downloaded deps now cache

CHANGELOG.md

@@ -1,7 +1,8 @@
 # Changelog
 
 ## v0.0.3
-- Added WIP manager to install Python binaries
+- Now manages and installs Python as required
+- Stores downloaded packages in a global cache
 - Can run console scripts specified in `pyproject.toml` directly, instead of just
 ones installed by dependencies
 - `pypackage reset` now cleans up the lock file

Cargo.toml

@@ -20,7 +20,7 @@ dirs = "^2.0.2"
 flate2 = "1.0.9"
 fs_extra = "^1.1.0"
 #lzma = "^0.2.2"
-#rust-lzma = "^0.4.0"
+rust-lzma = "^0.4.0"
 regex = "^1.1.9"
 reqwest = "^0.9.19"
 ring = "^0.16.5"

README.md

@@ -44,7 +44,7 @@ creates a folder with the basics
 
 
 ## Why add another Python manager?
-`Pipenv` and `Poetry` both address part of this problem. Goal: Faster and less finicky.
+`Pipenv` and `Poetry` both address part of this problem.
  Some reasons why this tool is different:
 
 - Its dependency resolution and locking is faster due to using a cached
@@ -214,6 +214,7 @@ a readme, pyproject.toml, .gitignore, and directory for code
 - `pypackage init` - Create a `pyproject.toml` file in an existing project directory. Pull info from
 `requirements.text` and `Pipfile` as required.
 - `pypackage reset` - Remove the environment, and uninstall all packages
+- `pypackage clear` - Clear the global cache of downloaded packages, in `~/python-installs/dependency-cache`
 - `pypackage -V` - Get the current version of this tool
 - `pypackage help` Get help, including a list of available commands
 
@@ -275,8 +276,6 @@ check for resolutions, then vary children as-required down the hierarchy. We don
 - Adding a dependency via the CLI with a specific version constraint, or extras.
 - Developer requirements
 - Packaging and publishing projects that use compiled extensions
-- Global package cache to avoid resolving and downloading the same package 
-for each project?
 - Download and install a Python version you specify, if not already installed
 
 
@@ -349,8 +348,9 @@ via a `deb`, `msi`, or `Cargo`, this should be set up automatically.
 
 # References
 - [PEP 582 - Python local packages directory](https://www.python.org/dev/peps/pep-0582/)
-- [Pep 518 - pyproject.toml](https://www.python.org/dev/peps/pep-518/)
+- [PEP 518 - pyproject.toml](https://www.python.org/dev/peps/pep-518/)
 - [Semantic versioning](https://semver.org/)
+- [PEP 440 -- Version Identification and Dependency Specification](https://www.python.org/dev/peps/pep-0440/)
 - [Specifying dependencies in Cargo](https://doc.rust-lang.org/cargo/reference/specifying-dependencies.html)
 - [Predictable dependency management blog entry](https://blog.rust-lang.org/2016/05/05/cargo-pillars.html)
 - [Blog on why Pyhon dependencies are hard to determine](https://dustingram.com/articles/2018/03/05/why-pypi-doesnt-know-dependencies/)

src/commands.rs

@@ -1,9 +1,8 @@
 use crate::util;
 use regex::Regex;
+//use std::process::Stdio;
 use std::{error::Error, fmt};
 use std::{path::PathBuf, process::Command};
-use std::io::Write;
-use std::process::Stdio;
 
 #[derive(Debug)]
 struct _ExecutionError {
@@ -86,8 +85,6 @@ pub(crate) fn create_venv2(
     Ok(())
 }
 
-
-
 pub(crate) fn run_python(
     bin_path: &PathBuf,
     lib_path: &PathBuf,
@@ -98,23 +95,23 @@ pub(crate) fn run_python(
     // Run this way instead of setting current_dir, so we can load files from the right place.
     // Running with .output() prevents the REPL from running, and .spawn() causes
     // permission errors when importing modules.
-//    let mut child = Command::new(bin_path.join("python"))
-//        .args(args)
-//        .stdin(Stdio::piped())
-//        .stdout(Stdio::piped())
-//        .spawn()?;
-//
-//    let stdin = child.stdin.as_mut().expect("failed to get stdin");
-//    stdin.write_all(b"test").expect("failed to write to stdin");
-//
-//    let output = child.wait_with_output().expect("failed to wait on child");
+    //    let mut child = Command::new(bin_path.join("python"))
+    //        .args(args)
+    //        .stdin(Stdio::piped())
+    //        .stdout(Stdio::piped())
+    //        .spawn()?;
+    //
+    //    let stdin = child.stdin.as_mut().expect("failed to get stdin");
+    //    stdin.write_all(b"test").expect("failed to write to stdin");
+    //
+    //    let output = child.wait_with_output().expect("failed to wait on child");
 
     //    let output = a.wait_with_output().expect("Failed to wait on sed");
     //    println!("{:?}", output.stdout.as_slice());
     //    Command::new(bin_path.join("python")).args(args).status()?;
     //    println!("ARGS: {:#?}", &args);
 
-        Command::new(bin_path.join("python")).args(args).status()?;
+    Command::new(bin_path.join("python")).args(args).status()?;
 
     //    Command::new(bin_path.join("python")).args(args).spawn()?;
     //    let output = Command::new(bin_path.join("python")).args(args).output()?;

src/install.rs

@@ -160,13 +160,17 @@ pub fn download_and_install_package(
     expected_digest: &str,
     lib_path: &PathBuf,
     bin_path: &PathBuf,
+    cache_path: &PathBuf,
     package_type: PackageType,
     rename: &Option<(u32, String)>,
 ) -> Result<(), reqwest::Error> {
     if !lib_path.exists() {
         fs::create_dir(lib_path).expect("Problem creating lib directory");
     }
-    let archive_path = lib_path.join(filename);
+    if !cache_path.exists() {
+        fs::create_dir(cache_path).expect("Problem creating cache directory");
+    }
+    let archive_path = cache_path.join(filename);
 
     // If the archive is already in the lib folder, don't re-download it. Note that this
     // isn't the usual flow, but may have some uses.
@@ -319,14 +323,6 @@ pub fn download_and_install_package(
     }
     setup_scripts(name, version, lib_path);
 
-    // Remove the archive
-    //    if fs::remove_file(&archive_path).is_err() {
-    //        util::abort(&format!(
-    //            "Problem removing this downloaded package: {:?}",
-    //            &archive_path
-    //        ));
-    //    } // todo
-
     Ok(())
 }
 

src/main.rs

@@ -127,6 +127,9 @@ Install packages from `pyproject.toml`, `pypackage.lock`, or speficied ones. Exa
     /// Remove the environment, and uninstall all packages
     #[structopt(name = "reset")]
     Reset,
+    /// Remove all cached packages.  Eg to free up hard drive space.
+    #[structopt(name = "clear")]
+    Clear,
     /// Run a CLI script like `ipython` or `black`. Note that you can simply run `pypackage black`
     /// as a shortcut.
     #[structopt(name = "run")] // We don't need to invoke this directly, but the option exists
@@ -629,6 +632,7 @@ fn find_best_release(
 fn sync_deps(
     bin_path: &PathBuf,
     lib_path: &PathBuf,
+    cache_path: &PathBuf,
     lock_packs: &[LockPackage],
     installed: &[(String, Version, Vec<String>)],
     os: Os,
@@ -708,6 +712,7 @@ fn sync_deps(
             &best_release.digests.sha256,
             lib_path,
             bin_path,
+            cache_path,
             package_type,
             rename,
         )
@@ -877,6 +882,7 @@ fn run_script(
 fn sync(
     bin_path: &PathBuf,
     lib_path: &PathBuf,
+    cache_path: &PathBuf,
     lockpacks: &[LockPackage],
     reqs: &[Req],
     os: Os,
@@ -994,6 +1000,7 @@ fn sync(
         //                &bin_path, &lib_path, &packages, &installed, &os, &py_vers, &resolved,
         &bin_path,
         &lib_path,
+        &cache_path,
         &updated_lock_packs,
         &installed,
         os,
@@ -1004,9 +1011,10 @@ fn sync(
 fn main() {
     let cfg_filename = "pyproject.toml";
     let lock_filename = "pypackage.lock";
-    //    let python_installs_dir = env::home_dir()
-    //        .expect("Problem finding home directory")
-    //        .join(".python-installs");
+    let python_installs_dir = dirs::home_dir()
+        .expect("Problem finding home directory")
+        .join(".python-installs");
+    let cache_path = python_installs_dir.join("dependency-cache");
 
     let mut cfg = Config::from_file(cfg_filename).unwrap_or_default();
     let opt = Opt::from_args();
@@ -1055,6 +1063,24 @@ fn main() {
             util::print_color("Reset complete", Color::Green);
             return;
         }
+        SubCommand::Switch { .. } => {
+            // Updates `pyproject.toml` with a new python version
+        }
+        SubCommand::Clear {} => {
+            if !cache_path.exists() {
+                fs::create_dir(&cache_path).expect("Problem creating cache directory");
+            }
+
+            for entry in fs::read_dir(&cache_path).expect("Problem reading cache path") {
+                if let Ok(entry) = entry {
+                    let path = entry.path();
+
+                    if path.is_file() {
+                        fs::remove_file(path).expect("Problem removing a cached file");
+                    }
+                };
+            }
+        }
         _ => (),
     }
 
@@ -1068,7 +1094,7 @@ fn main() {
         None => {
             // Ask the user, and write it to `pyproject.toml`.
             util::print_color(
-                "Please enter a Python version for this project:",
+                "Please enter the Python version for this project:",
                 Color::Magenta,
             );
             // todo: Utility fn for this type promp? Shared with prompt_alias.
@@ -1077,11 +1103,7 @@ fn main() {
                 .read_line(&mut input)
                 .expect("Unable to read user input for version");
 
-            let input = input
-                .chars()
-                .next()
-                .expect("Problem reading input")
-                .to_string();
+            input.pop(); // Remove trailing newline.
 
             let specified = match Version::from_str(&input) {
                 Ok(v) => v,
@@ -1126,28 +1148,6 @@ fn main() {
                 for this project.",
         ),
     }
-    // The version's not specified in the config - Ask the user which version.
-    //        None => {
-
-    //            match venvs.len() {
-
-    //            0 => {
-    //                let vers = create_venv(None, &pypackages_dir);
-    //                vers_path = pypackages_dir.join(&format!("{}.{}", vers.major, vers.minor));
-    //                py_vers = vers;
-    //            }
-    //            1 => {
-    //                vers_path = pypackages_dir.join(&format!("{}.{}", venvs[0].0, venvs[0].1));
-    //                py_vers = Version::new_short(venvs[0].0, venvs[0].1);
-    //            }
-    //            _ => abort(
-    //                "Multiple Python environments found
-    //                for this project; specify the desired one in `pyproject.toml`. Example:
-    //[tool.pypackage]
-    //py_version = \"3.7\"",
-    //            ),
-    //        },
-    //    };
 
     let lib_path = vers_path.join("lib");
     let bin_path = util::find_bin_path(&vers_path);
@@ -1190,6 +1190,7 @@ fn main() {
             sync(
                 &bin_path,
                 &lib_path,
+                &cache_path,
                 &lockpacks,
                 &updated_reqs,
                 os,
@@ -1224,6 +1225,7 @@ fn main() {
             sync(
                 &bin_path,
                 &lib_path,
+                &cache_path,
                 &lockpacks,
                 &updated_reqs,
                 os,
@@ -1247,11 +1249,12 @@ fn main() {
             return;
         }
         SubCommand::List {} => util::show_installed(&lib_path),
-        // We already handled init, and new, and reset
+        // We already handled these
         SubCommand::Init {} => (),
         SubCommand::New { .. } => (),
         SubCommand::Switch { .. } => (),
         SubCommand::Reset {} => (),
+        SubCommand::Clear {} => (),
     }
 }