site/docs/methods.html

<!DOCTYPE html>

<html>

<head>

<meta charset="utf-8" />
<meta name="generator" content="pandoc" />
<meta http-equiv="X-UA-Compatible" content="IE=EDGE" />


<meta name="author" content="Juniper L. Simonis" />

<meta name="date" content="2024-04-27" />

<title>Salvage Data Methods</title>

<script src="site_libs/jquery-3.6.0/jquery-3.6.0.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="site_libs/bootstrap-3.3.5/css/darkly.min.css" rel="stylesheet" />
<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<style>h1 {font-size: 34px;}
       h1.title {font-size: 38px;}
       h2 {font-size: 30px;}
       h3 {font-size: 24px;}
       h4 {font-size: 18px;}
       h5 {font-size: 16px;}
       h6 {font-size: 12px;}
       code {color: inherit; background-color: rgba(0, 0, 0, 0.04);}
       pre:not([class]) { background-color: white }</style>
<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>
<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />
<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>
<script src="site_libs/navigation-1.1/tabsets.js"></script>
<link href="site_libs/font-awesome-5.1.0/css/all.css" rel="stylesheet" />
<link href="site_libs/font-awesome-5.1.0/css/v4-shims.css" rel="stylesheet" />
<link rel="icon" type="image/png" href="images/favicon.png" />

<style type="text/css">
  code{white-space: pre-wrap;}
  span.smallcaps{font-variant: small-caps;}
  span.underline{text-decoration: underline;}
  div.column{display: inline-block; vertical-align: top; width: 50%;}
  div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
  ul.task-list{list-style: none;}
    </style>


<style type="text/css">
  code {
    white-space: pre;
  }
  .sourceCode {
    overflow: visible;
  }
</style>
<style type="text/css" data-origin="pandoc">
a.sourceLine { display: inline-block; line-height: 1.25; }
a.sourceLine { pointer-events: none; color: inherit; text-decoration: inherit; }
a.sourceLine:empty { height: 1.2em; }
.sourceCode { overflow: visible; }
code.sourceCode { white-space: pre; position: relative; }
div.sourceCode { margin: 1em 0; }
pre.sourceCode { margin: 0; }
@media screen {
div.sourceCode { overflow: auto; }
}
@media print {
code.sourceCode { white-space: pre-wrap; }
a.sourceLine { text-indent: -1em; padding-left: 1em; }
}
pre.numberSource a.sourceLine
  { position: relative; left: -4em; }
pre.numberSource a.sourceLine::before
  { content: attr(title);
    position: relative; left: -1em; text-align: right; vertical-align: baseline;
    border: none; pointer-events: all; display: inline-block;
    -webkit-touch-callout: none; -webkit-user-select: none;
    -khtml-user-select: none; -moz-user-select: none;
    -ms-user-select: none; user-select: none;
    padding: 0 4px; width: 4em;
    color: #aaaaaa;
  }
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa;  padding-left: 4px; }
div.sourceCode
  { background-color: #f8f8f8; }
@media screen {
a.sourceLine::before { text-decoration: underline; }
}
code span.al { color: #ef2929; } /* Alert */
code span.an { color: #8f5902; font-weight: bold; font-style: italic; } /* Annotation */
code span.at { color: #c4a000; } /* Attribute */
code span.bn { color: #0000cf; } /* BaseN */
code span.cf { color: #204a87; font-weight: bold; } /* ControlFlow */
code span.ch { color: #4e9a06; } /* Char */
code span.cn { color: #000000; } /* Constant */
code span.co { color: #8f5902; font-style: italic; } /* Comment */
code span.cv { color: #8f5902; font-weight: bold; font-style: italic; } /* CommentVar */
code span.do { color: #8f5902; font-weight: bold; font-style: italic; } /* Documentation */
code span.dt { color: #204a87; } /* DataType */
code span.dv { color: #0000cf; } /* DecVal */
code span.er { color: #a40000; font-weight: bold; } /* Error */
code span.ex { } /* Extension */
code span.fl { color: #0000cf; } /* Float */
code span.fu { color: #000000; } /* Function */
code span.im { } /* Import */
code span.in { color: #8f5902; font-weight: bold; font-style: italic; } /* Information */
code span.kw { color: #204a87; font-weight: bold; } /* Keyword */
code span.op { color: #ce5c00; font-weight: bold; } /* Operator */
code span.ot { color: #8f5902; } /* Other */
code span.pp { color: #8f5902; font-style: italic; } /* Preprocessor */
code span.sc { color: #000000; } /* SpecialChar */
code span.ss { color: #4e9a06; } /* SpecialString */
code span.st { color: #4e9a06; } /* String */
code span.va { color: #000000; } /* Variable */
code span.vs { color: #4e9a06; } /* VerbatimString */
code span.wa { color: #8f5902; font-weight: bold; font-style: italic; } /* Warning */

</style>
<script>
// apply pandoc div.sourceCode style to pre.sourceCode instead
(function() {
  var sheets = document.styleSheets;
  for (var i = 0; i < sheets.length; i++) {
    if (sheets[i].ownerNode.dataset["origin"] !== "pandoc") continue;
    try { var rules = sheets[i].cssRules; } catch (e) { continue; }
    var j = 0;
    while (j < rules.length) {
      var rule = rules[j];
      // check if there is a div.sourceCode rule
      if (rule.type !== rule.STYLE_RULE || rule.selectorText !== "div.sourceCode") {
        j++;
        continue;
      }
      var style = rule.style.cssText;
      // check if color or background-color is set
      if (rule.style.color === '' && rule.style.backgroundColor === '') {
        j++;
        continue;
      }
      // replace div.sourceCode by a pre.sourceCode rule
      sheets[i].deleteRule(j);
      sheets[i].insertRule('pre.sourceCode{' + style + '}', j);
    }
  }
})();
</script>


<link rel="stylesheet" href="styles.css" type="text/css" />


<style type = "text/css">
.main-container {
  max-width: 940px;
  margin-left: auto;
  margin-right: auto;
}
img {
  max-width:100%;
}
.tabbed-pane {
  padding-top: 12px;
}
.html-widget {
  margin-bottom: 20px;
}
button.code-folding-btn:focus {
  outline: none;
}
summary {
  display: list-item;
}
details > summary > p:only-child {
  display: inline;
}
pre code {
  padding: 0;
}
</style>


<style type="text/css">
.dropdown-submenu {
  position: relative;
}
.dropdown-submenu>.dropdown-menu {
  top: 0;
  left: 100%;
  margin-top: -6px;
  margin-left: -1px;
  border-radius: 0 6px 6px 6px;
}
.dropdown-submenu:hover>.dropdown-menu {
  display: block;
}
.dropdown-submenu>a:after {
  display: block;
  content: " ";
  float: right;
  width: 0;
  height: 0;
  border-color: transparent;
  border-style: solid;
  border-width: 5px 0 5px 5px;
  border-left-color: #cccccc;
  margin-top: 5px;
  margin-right: -10px;
}
.dropdown-submenu:hover>a:after {
  border-left-color: #adb5bd;
}
.dropdown-submenu.pull-left {
  float: none;
}
.dropdown-submenu.pull-left>.dropdown-menu {
  left: -100%;
  margin-left: 10px;
  border-radius: 6px 0 6px 6px;
}
</style>

<script type="text/javascript">
// manage active state of menu based on current page
$(document).ready(function () {
  // active menu anchor
  href = window.location.pathname
  href = href.substr(href.lastIndexOf('/') + 1)
  if (href === "")
    href = "index.html";
  var menuAnchor = $('a[href="' + href + '"]');

  // mark the anchor link active (and if it's in a dropdown, also mark that active)
  var dropdown = menuAnchor.closest('li.dropdown');
  if (window.bootstrap) { // Bootstrap 4+
    menuAnchor.addClass('active');
    dropdown.find('> .dropdown-toggle').addClass('active');
  } else { // Bootstrap 3
    menuAnchor.parent().addClass('active');
    dropdown.addClass('active');
  }

  // Navbar adjustments
  var navHeight = $(".navbar").first().height() + 15;
  var style = document.createElement('style');
  var pt = "padding-top: " + navHeight + "px; ";
  var mt = "margin-top: -" + navHeight + "px; ";
  var css = "";
  // offset scroll position for anchor links (for fixed navbar)
  for (var i = 1; i <= 6; i++) {
    css += ".section h" + i + "{ " + pt + mt + "}\n";
  }
  style.innerHTML = "body {" + pt + "padding-bottom: 40px; }\n" + css;
  document.head.appendChild(style);
});
</script>

<!-- tabsets -->

<style type="text/css">
.tabset-dropdown > .nav-tabs {
  display: inline-table;
  max-height: 500px;
  min-height: 44px;
  overflow-y: auto;
  border: 1px solid #ddd;
  border-radius: 4px;
}

.tabset-dropdown > .nav-tabs > li.active:before, .tabset-dropdown > .nav-tabs.nav-tabs-open:before {
  content: "\e259";
  font-family: 'Glyphicons Halflings';
  display: inline-block;
  padding: 10px;
  border-right: 1px solid #ddd;
}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li.active:before {
  content: "\e258";
  font-family: 'Glyphicons Halflings';
  border: none;
}

.tabset-dropdown > .nav-tabs > li.active {
  display: block;
}

.tabset-dropdown > .nav-tabs > li > a,
.tabset-dropdown > .nav-tabs > li > a:focus,
.tabset-dropdown > .nav-tabs > li > a:hover {
  border: none;
  display: inline-block;
  border-radius: 4px;
  background-color: transparent;
}

.tabset-dropdown > .nav-tabs.nav-tabs-open > li {
  display: block;
  float: none;
}

.tabset-dropdown > .nav-tabs > li {
  display: none;
}
</style>

<!-- code folding -->


<style type="text/css">

#TOC {
  margin: 25px 0px 20px 0px;
}
@media (max-width: 768px) {
#TOC {
  position: relative;
  width: 100%;
}
}

@media print {
.toc-content {
  /* see https://github.com/w3c/csswg-drafts/issues/4434 */
  float: right;
}
}

.toc-content {
  padding-left: 30px;
  padding-right: 40px;
}

div.main-container {
  max-width: 1200px;
}

div.tocify {
  width: 20%;
  max-width: 260px;
  max-height: 85%;
}

@media (min-width: 768px) and (max-width: 991px) {
  div.tocify {
    width: 25%;
  }
}

@media (max-width: 767px) {
  div.tocify {
    width: 100%;
    max-width: none;
  }
}

.tocify ul, .tocify li {
  line-height: 20px;
}

.tocify-subheader .tocify-item {
  font-size: 0.90em;
}

.tocify .list-group-item {
  border-radius: 0px;
}

.tocify-subheader {
  display: inline;
}
.tocify-subheader .tocify-item {
  font-size: 0.95em;
}

</style>


</head>

<body>


<div class="container-fluid main-container">


<!-- setup 3col/9col grid for toc_float and main content  -->
<div class="row">
<div class="col-xs-12 col-sm-4 col-md-3">
<div id="TOC" class="tocify">
</div>
</div>

<div class="toc-content col-xs-12 col-sm-8 col-md-9">


<div class="navbar navbar-default  navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-bs-toggle="collapse" data-target="#navbar" data-bs-target="#navbar">
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="index.html">Delta Fish Salvage</a>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav">
        <li>
  <a href="index.html">
    <span class="fa fa-home"></span>
     
  </a>
</li>
<li>
  <a href="summary.html">Summary</a>
</li>
<li>
  <a href="methods.html">Methods</a>
</li>
      </ul>
      <ul class="nav navbar-nav navbar-right">
        <li>
  <a href="https://www.github.com/dapperstats/salvage">
    <span class="fa fa-github"></span>
     
  </a>
</li>
<li>
  <a href="https://www.twitter.com/dapperstats">
    <span class="fa fa-twitter"></span>
     
  </a>
</li>
<li>
  <a href="https://www.dapperstats.com">DAPPER Stats</a>
</li>
      </ul>
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

<div id="header">


<h1 class="title toc-ignore">Salvage Data Methods</h1>
<h4 class="author">Juniper L. Simonis</h4>
<h4 class="date">April 27, 2024</h4>

</div>


<p><br></p>
<div id="data-access" class="section level2">
<h2>Data Access</h2>
<p><br></p>
<div id="retrieve-the-salvage-database" class="section level3">
<h3>Retrieve the Salvage Database</h3>
<p>The bulk of the data access protocol involves converting the <code>.accdb</code> salvagae database file on the remote <a href="ftp://ftp.dfg.ca.gov/salvage/">ftp</a> to a local set of <code>.csv</code> files named by the tables in the database. We accomplish this in two lines of code by pulling and then running a stable <a href="https://www.docker.com"><code>Docker</code></a> <a href="https://www.docker.com/resources/what-container">software container</a> that contains a set of <code>bash</code> scripts designed specifically for this task. The specific image used for data access is called <a href="https://hub.docker.com/r/dapperstats/accessor"><code>accessor</code></a>, is freely available on <a href="https://hub.docker.com/">Docker Hub</a>, and has default setting configured for the salvage database. Code for the construction of the <code>accessor</code> image is available in its <a href="https://www.github.com/dapperstats/accessor">repository</a>.</p>
<p>For accessability and reproducibility, we provide an <a href="https://github.com/dapperstats/salvage/blob/master/data">up-to-date version of the salvage data</a> as <code>.csv</code>s from the “current” (1993 - Present) salvage database file (<code>Salvage_data_FTP.accdb</code>). The data can be downloaded via various methods from the repository, including from the website.</p>
<p>Updates to the data are executed via <a href="https://docs.travis-ci.com/user/cron-jobs/"><code>cron</code> jobs</a> on <a href="https://travis-ci.org/dapperstats/salvage"><code>travis-ci</code></a> and pushed to GitHub as <a href="https://github.com/dapperstats/salvage/releases">tagged Releases</a>.</p>
<p><br></p>
<div id="build-a-salvage-container" class="section level4">
<h4>Build A Salvage Container</h4>
<p>To use the current image to generate an up-to-date container with data for yourself</p>
<ol style="list-style-type: decimal">
<li><a href="https://docs.docker.com/get-docker/">Install Docker</a>
<ul>
<li>Specific instructions vary depending on OS</li>
</ul></li>
<li>Open up a docker-ready terminal</li>
<li>Download the image</li>
</ol>
<div class="sourceCode" id="cb1"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb1-1" title="1"><span class="fu">sudo</span> docker pull dapperstats/salvage<span class="kw">`</span></a></code></pre></div>
<ol start="4" style="list-style-type: decimal">
<li>Build the container</li>
</ol>
<div class="sourceCode" id="cb2"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb2-1" title="1"><span class="fu">sudo</span> docker container run -ti --name salvage dapperstats/salvage<span class="kw">`</span></a></code></pre></div>
<ol start="5" style="list-style-type: decimal">
<li>Copy the data out from the container</li>
</ol>
<div class="sourceCode" id="cb3"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb3-1" title="1"><span class="fu">sudo</span> docker cp salvage:/data .<span class="kw">`</span></a></code></pre></div>
<p><br></p>
</div>
</div>
<div id="bring-the-data-into-r" class="section level3">
<h3>Bring the Data into R</h3>
<p>An additional conversion makes the data available in <a href="https://www.r-project.org/"><code>R</code></a> as a <code>list</code> of <code>data.frames</code> that is directly analagous to the <code>.accdb</code> database of tables.</p>
<p>The reading into R is conducted via functions included in the <code>r_functions.R</code> script in the <a href="https://hub.docker.com/r/dapperstats/accessor"><code>accessor</code> image</a> and available in the <a href="https://github.com/dapperstats/accessor/tree/master/scripts">public code repository</a>.</p>
<p><br></p>
<div id="within-a-salvage-container" class="section level4">
<h4>Within a Salvage Container</h4>
<p>Building on the list above, you can leverage the <code>r_script.R</code> script included in the image, which sources the <code>r_functions.R</code> script and loads the database in as an <code>R</code> object named <code>database</code>. Docker provides ample access and avenues to run R within the container. For example, the <code>docker exec</code> command opens a full API for running within the top (read/write) layer of the container, allowing an endless supply of R code to be included within a single character input:</p>
<ol start="6" style="list-style-type: decimal">
<li>Run a <code>bash</code> <code>R</code> script from the command line</li>
</ol>
<div class="sourceCode" id="cb4"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb4-1" title="1"><span class="fu">sudo</span> docker exec -i salvage R -e <span class="st">&quot;source(&#39;scripts/r_script.R&#39;); &lt;additional R code&gt;&quot;</span></a></code></pre></div>
<p>You can copy your own scripts <em>into</em> the image and then run them from that environment:</p>
<ol start="7" style="list-style-type: decimal">
<li>Copying a script into the image</li>
</ol>
<div class="sourceCode" id="cb5"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb5-1" title="1"><span class="fu">sudo</span> docker cp path/to/my_r_script.R salvage:/scripts</a></code></pre></div>
<ol start="8" style="list-style-type: decimal">
<li>Running the script in the image</li>
</ol>
<div class="sourceCode" id="cb6"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb6-1" title="1"><span class="fu">sudo</span> docker exec -i salvage R -e <span class="st">&quot;source(&#39;scripts/r_script.R&#39;); source(&#39;scripts/my_r_script.R&#39;)&quot;</span></a></code></pre></div>
<p>Note that we are running the main <code>r_script.R</code> first still here; that script does not save any files externally, so the R session in <strong>6.</strong> is gone when we run <strong>8.</strong>. For the sake of simplifying the command line call, it is therefore recommended that expanded uses follow <strong>8.</strong> and use <code>my_r_script.R</code> as a hub file that directs all of your specific functions, including files saved out from R. For simplicity, saving out all files into a single folder, e.g. <code>output</code> allows a single docker command for retrieving the results from the top layer of the container:</p>
<ol start="9" style="list-style-type: decimal">
<li>Copy the output out from the container</li>
</ol>
<div class="sourceCode" id="cb7"><pre class="sourceCode bash"><code class="sourceCode bash"><a class="sourceLine" id="cb7-1" title="1"><span class="fu">sudo</span> docker cp salvage:/output .</a></code></pre></div>
<p><br></p>
</div>
<div id="working-in-an-open-r-session" class="section level4">
<h4>Working in an Open R Session</h4>
<p>Alternatively, the functions are written in only base R, so they <strong>should</strong> be reproducibly functional outside the image (in an open session).</p>
<p>Within an instance of <code>R</code>, navigate to where you have read the data out from the container into/where this code repository located and source <code>r_script.R</code>:</p>
<ol start="6" style="list-style-type: decimal">
<li>Load the functions and read in the data</li>
</ol>
<div class="sourceCode" id="cb8"><pre class="sourceCode r"><code class="sourceCode r"><a class="sourceLine" id="cb8-1" title="1"><span class="kw">source</span>(<span class="st">&quot;scripts/r_script.R&quot;</span>)</a></code></pre></div>
<p>The resulting <code>database</code> object is a named <code>list</code> of the database’s tables, ready for analyses.</p>
<p><br></p>
</div>
</div>
</div>
<div id="data-preparation" class="section level2">
<h2>Data Preparation</h2>
<p>Data preparation code is <em>in development</em>!</p>
<p>Having brought the data into R as-is, we can now prepare them for summaries and analyses. We use the functions included in the <a href="https://github.com/dapperstats/salvage/blob/master/scripts/salvage_functions.R"><code>salvage_functions.R</code> R script</a>, which is included within the and <a href="https://hub.docker.com/r/dapperstats/salvage"><code>salvage</code> docker image</a>, which provides a stable runtime environment for the analyses and output generation (including website rendering).</p>
<p><br></p>
</div>
<div id="continuous-deployment" class="section level2">
<h2>Continuous Deployment</h2>
<p>The data and output are updated daily via <a href="https://docs.travis-ci.com/user/cron-jobs/"><code>cron</code> jobs</a> on <a href="https://travis-ci.org/dapperstats/salvage"><code>travis-ci</code></a> with a recipe (a.k.a. job lifecycle) described by the <a href="https://github.com/dapperstats/salvage/blob/master/.travis.yml"><code>.travis.yml</code> file</a>.</p>
<p><br> <br></p>
</div>

<p>
  Copyright &copy; 2020 DAPPER Stats &nbsp; &nbsp;
  <a href = "https://github.com/dapperstats/salvage/blob/master/LICENSE">
    <img alt="GitHub" src="https://img.shields.io/github/license/dapperstats/salvage?color=blue">
  </a> &nbsp; &nbsp;
  <a href="https://doi.org/10.5281/zenodo.3628045">
    <img src="https://zenodo.org/badge/DOI/10.5281/zenodo.3628045.svg" alt="DOI">
  </a>
</p>


</div>
</div>

</div>

<script>

// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
  $('tr.odd').parent('tbody').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
  bootstrapStylePandocTables();
});


</script>

<!-- tabsets -->

<script>
$(document).ready(function () {
  window.buildTabsets("TOC");
});

$(document).ready(function () {
  $('.tabset-dropdown > .nav-tabs > li').click(function () {
    $(this).parent().toggleClass('nav-tabs-open');
  });
});
</script>

<!-- code folding -->

<script>
$(document).ready(function ()  {

    // temporarily add toc-ignore selector to headers for the consistency with Pandoc
    $('.unlisted.unnumbered').addClass('toc-ignore')

    // move toc-ignore selectors from section div to header
    $('div.section.toc-ignore')
        .removeClass('toc-ignore')
        .children('h1,h2,h3,h4,h5').addClass('toc-ignore');

    // establish options
    var options = {
      selectors: "h1,h2,h3,h4,h5",
      theme: "bootstrap3",
      context: '.toc-content',
      hashGenerator: function (text) {
        return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_');
      },
      ignoreSelector: ".toc-ignore",
      scrollTo: 0
    };
    options.showAndHide = false;
    options.smoothScroll = true;

    // tocify
    var toc = $("#TOC").tocify(options).data("toc-tocify");
});
</script>

<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>