Merge branch 'develop'

circle.yml

@@ -18,9 +18,15 @@ test:
                 - lein test:
                     timeout: 1800
                 - lein clean && lein with-profile uberjar uberjar
-                - docker build -t onyx/onyx-dashboard:$CIRCLE_SHA1 . | tee docker-build.log
-                - docker tag $(tail -1 docker-build.log | cut -d ' ' -f 3) onyx/onyx-dashboard:latest
-                - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
-                - docker push onyx/onyx-dashboard:$CIRCLE_SHA1
-                - docker push onyx/onyx-dashboard:latest
                 - cp target/onyx-dashboard.jar $CIRCLE_ARTIFACTS
+
+deployment:
+        production:
+                branch: master
+                commands:
+                        - docker build -t onyx/onyx-dashboard:$CIRCLE_SHA1 . | tee docker-build.log
+                        - docker tag $(tail -1 docker-build.log | cut -d ' ' -f 3) onyx/onyx-dashboard:latest
+                        - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+                        - docker push onyx/onyx-dashboard:$CIRCLE_SHA1
+                        - docker push onyx/onyx-dashboard:latest
+

env/dev/cljs/onyx_dashboard/dev.cljs

@@ -10,7 +10,7 @@
 (def is-dev? true)
 
 (figwheel/watch-and-reload
-  :websocket-url "ws://localhost:3449/figwheel-ws"
+  :websocket-url "ws://localhost:3428/figwheel-ws"
   :jsload-callback (fn []
                      (core/main is-dev?)))
 

project.clj

@@ -1,4 +1,4 @@
-(defproject onyx-dashboard "0.5.2.1"
+(defproject onyx-dashboard "0.5.2.2"
   :description "Dashboard for the Onyx distributed computation system"
   :url "http://github.com/lbradstreet/onyx-dashboard"
   :license {:name "Eclipse Public License"
@@ -31,6 +31,7 @@
                  [fipp "0.5.2"]
                  [environ "1.0.0"]
                  [http-kit "2.1.19"]
+                 [shoreleave/shoreleave-browser "0.3.0"]
                  ; make this explicit to fix uberjar?
                  [potemkin "0.3.11"]
                  [org.omcljs/om "0.8.8"]
@@ -74,7 +75,7 @@
                    :plugins [[lein-figwheel "0.2.3-SNAPSHOT"]]
 
                    :figwheel {:http-server-root "public"
-                              :server-port 3449
+                              :server-port 3428
                               :css-dirs ["resources/public/css"]}
 
                    :env {:is-dev true}

resources/index.html

@@ -6,6 +6,23 @@
     <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css">
     <link href="css/style.css" rel="stylesheet" type="text/css">
     <link href="css/ankha.css" rel="stylesheet" type="text/css">
+    <link rel="apple-touch-icon" sizes="57x57" href="favicons/apple-icon-57x57.png">
+    <link rel="apple-touch-icon" sizes="60x60" href="favicons/apple-icon-60x60.png">
+    <link rel="apple-touch-icon" sizes="72x72" href="favicons/apple-icon-72x72.png">
+    <link rel="apple-touch-icon" sizes="76x76" href="favicons/apple-icon-76x76.png">
+    <link rel="apple-touch-icon" sizes="114x114" href="favicons/apple-icon-114x114.png">
+    <link rel="apple-touch-icon" sizes="120x120" href="favicons/apple-icon-120x120.png">
+    <link rel="apple-touch-icon" sizes="144x144" href="favicons/apple-icon-144x144.png">
+    <link rel="apple-touch-icon" sizes="152x152" href="favicons/apple-icon-152x152.png">
+    <link rel="apple-touch-icon" sizes="180x180" href="favicons/apple-icon-180x180.png">
+    <link rel="icon" type="image/png" sizes="192x192"  href="favicons/android-icon-192x192.png">
+    <link rel="icon" type="image/png" sizes="32x32" href="favicons/favicon-32x32.png">
+    <link rel="icon" type="image/png" sizes="96x96" href="favicons/favicon-96x96.png">
+    <link rel="icon" type="image/png" sizes="16x16" href="favicons/favicon-16x16.png">
+    <link rel="manifest" href="favicons/manifest.json">
+    <meta name="msapplication-TileColor" content="#ffffff">
+    <meta name="msapplication-TileImage" content="favicons/ms-icon-144x144.png">
+    <meta name="theme-color" content="#ffffff">
   </head>
   <body>
     <div id="app"></div>

resources/public/favicons/browserconfig.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig><msapplication><tile><square70x70logo src="/ms-icon-70x70.png"/><square150x150logo src="/ms-icon-150x150.png"/><square310x310logo src="/ms-icon-310x310.png"/><TileColor>#ffffff</TileColor></tile></msapplication></browserconfig>

resources/public/favicons/manifest.json

@@ -0,0 +1,41 @@
+{
+ "name": "App",
+ "icons": [
+  {
+   "src": "\/android-icon-36x36.png",
+   "sizes": "36x36",
+   "type": "image\/png",
+   "density": "0.75"
+  },
+  {
+   "src": "\/android-icon-48x48.png",
+   "sizes": "48x48",
+   "type": "image\/png",
+   "density": "1.0"
+  },
+  {
+   "src": "\/android-icon-72x72.png",
+   "sizes": "72x72",
+   "type": "image\/png",
+   "density": "1.5"
+  },
+  {
+   "src": "\/android-icon-96x96.png",
+   "sizes": "96x96",
+   "type": "image\/png",
+   "density": "2.0"
+  },
+  {
+   "src": "\/android-icon-144x144.png",
+   "sizes": "144x144",
+   "type": "image\/png",
+   "density": "3.0"
+  },
+  {
+   "src": "\/android-icon-192x192.png",
+   "sizes": "192x192",
+   "type": "image\/png",
+   "density": "4.0"
+  }
+ ]
+}

src/clj/onyx_dashboard/onyx_deployment.clj

@@ -167,7 +167,7 @@
         (let [entry (extensions/read-log-entry log position)]
           (if-let [new-replica (apply-log-entry send-fn! tracking-id entry replica)] 
             (let [diff (extensions/replica-diff entry replica new-replica)
-                  new-incomplete-jobs (set (common/incomplete-jobs replica))]
+                  new-incomplete-jobs (set (common/incomplete-jobs new-replica))]
               (send-job-statuses send-fn! tracking-id incomplete-jobs new-incomplete-jobs)
               (log-notifications send-fn! new-replica diff log entry tracking-id)
               (send-log-entry send-fn! tracking-id entry)

src/cljs/onyx_dashboard/components/deployment.cljs

@@ -5,9 +5,13 @@
             [om-bootstrap.panel :as p]
             [om-bootstrap.button :as b]
             [om-bootstrap.table :as t]
+            [om-bootstrap.grid :as g]
+            [shoreleave.browser.blob :as blob]
             [onyx-dashboard.components.ui-elements :refer [section-header-collapsible]]
             [cljs.core.async :as async :refer [put!]]
-            [cljsjs.moment]))
+            [cljsjs.moment]
+            [cljs.core.async :as async :refer [<! >! put! chan]])
+  (:require-macros [cljs.core.async.macros :as asyncm :refer [go-loop]]))
 
 (defcomponent peer-table [peers owner]
   (render [_]
@@ -47,15 +51,107 @@
 
 (defcomponent deployment-peers [deployment owner]
   (render [_] 
-          (dom/div
-            (p/panel
-              {:header (om/build section-header-collapsible {:text "Deployment Peers"} {})
-               :collapsible? true
-               :bs-style "primary"}
-              (if (and (:id deployment) 
-                       (:up? deployment)) 
-                (om/build peer-table (:peers deployment) {})
-                (dom/div "Deployment has no pulse."))))))
+          (p/panel
+            {:header (om/build section-header-collapsible {:text "Deployment Peers"} {})
+             :collapsible? true
+             :bs-style "primary"}
+            (if (and (:id deployment) 
+                     (:up? deployment)) 
+              (om/build peer-table (:peers deployment) {})
+              (dom/div "Deployment has no pulse.")))))
+
+(defn strip-catalog [catalog task-rename]
+  (mapv (fn [entry]
+          (-> entry 
+              (update-in [:onyx/name] task-rename)
+              (select-keys [:onyx/name :onyx/type :onyx/ident 
+                            :onyx/medium :onyx/consumption 
+                            :onyx/batch-size]))) 
+          catalog))
+
+(defn replace-in-workflow [workflow translation]
+  (mapv (fn [[from to]]
+          [(translation from) (translation to)])
+        workflow))
+
+(defn workflow->task-rename-map [workflow]
+  (zipmap (distinct (flatten workflow)) 
+          (map (comp keyword str) (range))))
+
+(defn publicise-jobs [jobs]
+  (mapv (fn [{:keys [workflow catalog] :as job}]
+          (let [task-rename (workflow->task-rename-map workflow)] 
+            (-> job
+                (dissoc :pretty-workflow
+                        :pretty-catalog
+                        :tracking-id)
+                (update-in [:workflow] replace-in-workflow task-rename)
+                (update-in [:catalog] strip-catalog task-rename))))
+        (vals jobs)))
+
+(defn entries->log-dump [entries]
+  (vec (sort-by :message-id (vals entries))))
+
+(defn serialize [v]
+  (vector (pr-str v)))
+
+(defcomponent download-with-filename 
+  "Gross hack to enable blobs to be saved with a particular filename.
+  Unfortunately, the only way to provide a filename is with a link that gets clicked.
+  window.open on the blob object-url doens't currently work"
+  [{:keys [data filename]} owner {:keys [parent-ch]}]
+  (did-mount [_]
+             (.click (om/get-node owner))
+             (put! parent-ch :done))
+  (render [_]
+          (dom/a {:href (blob/object-url! 
+                          (blob/blob (serialize data)
+                                     "application/octet-stream"))
+                  :download filename} 
+                 "Save")))
+
+(defcomponent deployment-log-dump [{:keys [entries jobs id] :as deployment} owner]
+  (init-state [_] 
+              {:download-type nil :download-ch (chan)})
+  (will-mount [_]
+              (go-loop []
+                       (let [msg (<! (om/get-state owner :download-ch))]
+                         (if (= msg :done)
+                           (om/set-state! owner :download-type nil)))))
+  (render-state [_ {:keys [download-type download-ch]}] 
+                (dom/div
+                  (case download-type 
+                    :raw-dump (om/build download-with-filename 
+                                        {:data {:jobs jobs
+                                                :log (entries->log-dump entries)}
+                                         :filename (str "dump_" id "_raw.edn")}
+                                        {:opts {:parent-ch download-ch}})
+                    :stripped-dump (om/build download-with-filename 
+                                             {:data {:jobs (publicise-jobs jobs)
+                                                     :log (entries->log-dump entries)}
+                                              :filename (str "dump_" id "_stripped.edn")}
+                                             {:opts {:parent-ch download-ch}})
+                    (dom/div))
+                  (p/panel
+                    {:header "Deployment Log Dump" 
+                     :collapsible? true
+                     :bs-style "primary"}
+                    (t/table {:striped? true :bordered? false :condensed? true :hover? true}
+                             (dom/thead (dom/tr (dom/th "Type") (dom/th)))
+                             (dom/tbody
+                               (dom/tr 
+                                 (dom/td "Raw")
+                                 (dom/td
+                                   (dom/a {:on-click (fn [_] (om/set-state! owner :download-type :raw-dump))} 
+                                          "Save"))) 
+                               (dom/tr 
+                                 (dom/td "Stripped of catalog parameterisations")
+                                 (dom/td
+                                   (dom/a {:on-click (fn [_] (om/set-state! owner :download-type :stripped-dump))}
+                                          "Save")))))
+                    "WARNING: We make a best effort attempt to strip catalog parameterisation and task names. " 
+                    "If these may include private information, please inspect the :jobs value "
+                    "in the dump before making it public."))))
 
 
 (defcomponent select-deployment [{:keys [deployments deployment]} owner]

src/cljs/onyx_dashboard/components/log.cljs

@@ -58,6 +58,12 @@
      :current-page (- num-pages (Math/ceil (* num-pages (/ start-index (count entries)))))
      :num-pages num-pages}))
 
+(defn entry-about-job? [job-id {:keys [args] :as entry}]
+  (or (= job-id (:job args))
+      (= job-id (:id args))))
+
+
+
 (defcomponent log-entries-pager [{:keys [job-filter entries] :as log} owner]
   (init-state [_]
               {:entry-index nil
@@ -69,47 +75,48 @@
                          (om/set-state! owner :visible-entry entry))
                        (recur)))
   (render-state [_ {:keys [entry-index visible-entry entry-ch]}]
-                (let [filtered-entries (vec
-                                         (sort-by :message-id
-                                                  (cond->> (vals entries)
-                                                    job-filter (filter (comp (partial = job-filter)
-                                                                             :job
-                                                                             :args)))))] 
-                  (if (empty? filtered-entries)
-                    (dom/div {} "")
-                    (let [max-id (dec (count filtered-entries))
-                          current-index (or entry-index max-id)
-                          {:keys [num-pages current-page displayed-entries]} (pagination-info filtered-entries current-index)
-                          pagination-start (max (- current-page 
-                                                   (/ num-pages-to-show 2)) 
-                                                0)
-                          pages-window (take num-pages-to-show (range pagination-start num-pages))
-                          pages-to-show (cond-> pages-window
-                                          (not= 0 (first pages-window)) (conj 0)
-                                          (not= (dec num-pages) 
-                                                (last pages-window)) (concat [(dec num-pages)]))
-                          change-index (fn [index e]
-                                         (om/set-state! owner :entry-index index)    
-                                         (.preventDefault e)) 
-                          previous-handler (partial change-index 
-                                                    (let [new-index (min max-id 
-                                                                         (+ current-index entries-per-page))]
-                                                      (if (= new-index max-id)
-                                                        nil
-                                                        new-index)))
-                          next-handler (partial change-index
-                                                (max 0 (- current-index entries-per-page)))]
-                      (dom/div
-                        (if visible-entry 
-                          (om/build log-entry-modal (entries visible-entry) {:opts {:entry-ch entry-ch}}))
-                        (p/panel {:header (om/build section-header-collapsible 
-                                                    {:text (str "Raw Cluster Activity" 
-                                                                (if job-filter (str " - Job " job-filter)))}
-                                                    {})
-                                  :collapsible? true
-                                  :bs-style "primary"}
+                (let [filtered-entries (vec (cond->> (vals entries)
+                                              job-filter (filter (partial entry-about-job? job-filter))
+                                              true (sort-by :message-id)))] 
+                  (p/panel {:header (om/build section-header-collapsible 
+                                              {:text (str "Raw Cluster Activity" 
+                                                          (if job-filter (str " - Job " job-filter)))}
+                                              {})
+                            :collapsible? true
+                            :bs-style "primary"}
+                           (if (empty? filtered-entries)
+                             (dom/div "No log entries found.")
+                             (let [max-id (dec (count filtered-entries))
+                                   current-index (or entry-index max-id)
+                                   {:keys [num-pages current-page displayed-entries]} (pagination-info filtered-entries current-index)
+                                   pagination-start (max (- current-page 
+                                                            (/ num-pages-to-show 2)) 
+                                                         0)
+                                   pages-window (take num-pages-to-show (range pagination-start num-pages))
+                                   pages-to-show (cond-> pages-window
+                                                   (not= 0 (first pages-window)) (conj 0)
+                                                   (not= (dec num-pages) 
+                                                         (last pages-window)) (concat [(dec num-pages)]))
+                                   change-index (fn [index e]
+                                                  (om/set-state! owner :entry-index index)    
+                                                  (.preventDefault e)) 
+                                   previous-handler (partial change-index 
+                                                             (let [new-index (min max-id 
+                                                                                  (+ current-index entries-per-page))]
+                                                               (if (= new-index max-id)
+                                                                 nil
+                                                                 new-index)))
+                                   next-handler (partial change-index
+                                                         (max 0 (- current-index entries-per-page)))]
+                               (dom/div
+
+                                 (if visible-entry 
+                                   (om/build log-entry-modal (entries visible-entry) {:opts {:entry-ch entry-ch}}))
+
+
                                  (dom/div
                                    (om/build log-entries-table displayed-entries {:opts {:entry-ch entry-ch}})
+
                                    (pg/pagination {}
                                                   (pg/previous 
                                                     (if (zero? current-page) 
@@ -128,4 +135,7 @@
                                                   (pg/next 
                                                     (if (= current-page (dec num-pages))
                                                       {:disabled? true}
-                                                      {:on-click next-handler})))))))))))
+                                                      {:on-click next-handler}))))
+
+
+                                 )))))))