-
Notifications
You must be signed in to change notification settings - Fork 8
/
jplotter.py
3003 lines (2722 loc) · 131 KB
/
jplotter.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Interface to the plotiterator.
#
# Well... it's a sort of a high level interface so the user does not
# have to worry about having to translate DATA_DESC_IDs etc.
#
# The info will be presented in a sort of HRF (Human Readable Format)
#
# Rewrite from Ancient glish code to Python by H. Verkouter
# Dec 2012
#
# $Id: jplotter.py,v 1.73 2017/08/29 11:27:36 jive_cc Exp $
#
# $Log: jplotter.py,v $
# Revision 1.73 2017/08/29 11:27:36 jive_cc
# HV: * Added post-processing-pre-display filtering of datasets to be plotted
# This allows e.g. per subplot (think: amp+phase) selection of things to
# plot - users wanted to display phase of parallel pols but amp of
# everything:
# pt anpchan; filter phase: p in [ll,rr];
#
# Revision 1.72 2017-04-26 12:27:17 jive_cc
# HV: * ZsoltP + BenitoM: "in amplitude+phase plots it would be much
# better to have phase drawn with points and amplitude with lines"
# So now it is possible to change the drawing style of subplots
# individually in stead of just globally for all subplots.
# See "help draw" for examples.
#
# Revision 1.71 2017-04-24 10:10:26 jive_cc
# HV: * Based on BenitoM's comment added the 'x' and 'y' commands as separate
# commands such that you can now type "help x" or "help y" and the
# axis-scaling help comes up like any sane person would expect
#
# Revision 1.70 2017-03-20 14:54:55 jive_cc
# HV: * The "add column name" patch broke UV plotting. Now it should work OK
#
# Revision 1.69 2017-03-10 13:32:28 jive_cc
# HV: * feature request (IlsevB) "put data column name in plot metadata"
#
# Revision 1.68 2017-01-27 13:50:28 jive_cc
# HV: * jplotter.py: small edits
# - "not refresh(e)" => "refresh(e); if not e.plots ..."
# - "e.rawplots.XXX" i.s.o. "e.plots.XXX"
# * relatively big overhaul: in order to force (old) pyrap to
# re-read tables from disk all table objects must call ".close()"
# when they're done.
# Implemented by patching the pyrap.tables.table object on the fly
# with '__enter__' and '__exit__' methods (see "ms2util.opentable(...)")
# such that all table access can be done in a "with ..." block:
# with ms2util.opentable(...) as tbl:
# tbl.getcol('DATA') # ...
# and then when the with-block is left, tbl gets automagically closed
#
# Revision 1.67 2016-01-11 15:30:30 jive_cc
# HV: * channel averaging display 'crashed' if no channel selection was
# present (chanSel==None => no channels selected => i.e. all channels)
#
# Revision 1.66 2016-01-11 13:27:19 jive_cc
# HV: * "goto page N" was broken in interactive mode. Now just type
# <digits><enter> to go to a specific page
#
# Revision 1.65 2015-12-09 07:02:11 jive_cc
# HV: * big change! the plotiterators now return a one-dimensional dict
# of label => dataset. The higher level code reorganizes them
# into plots, based on the 'new plot' settings. Many wins in this one:
# - the plotiterators only have to index one level in stead of two
# - when the 'new plot' setting is changed, we don't have to read the
# data from disk again [this is a *HUGE* improvement, especially for
# larger data sets]
# - the data set expression parser is simpler, it works on the
# one-dimensional 'list' of data sets and it does not have to
# flatten/unflatten any more
# * The code to deal with refreshing the plots has been rewritten a bit
# such that all necessary steps (re-organizing raw plots into plots,
# re-running the label processing, re-running the post processing,
# re-running the min/max processing) are executed only once; when
# necessary. And this code is now shared between the "pl" command and
# the "load/store" commands.
#
# Revision 1.64 2015-10-02 14:47:56 jive_cc
# HV: * multi subband plotting:
# - default is now 'False' in stead of 'None'
# - multi command does not accept "1" or "0" any more
# - multi command accepts t(rue) or f(alse)
#
# Revision 1.63 2015-09-23 12:28:36 jive_cc
# HV: * Lorant S. requested sensible requests (ones that were already in the
# back of my mind too):
# - option to specify the data column
# - option to not reorder the spectral windows
# Both options are now supported by the code and are triggered by
# passing options to the "ms" command
#
# Revision 1.62 2015-09-21 11:36:22 jive_cc
# HV: * The big push. The 'command.py' interface long left something
# to be desired if a command specified an 'args()' method to transform
# the line of input into multiple arguments and call the callback function
# with those arguments [hint: it wouldn't].
# Now the command.py object calls the callback as:
# "cmd.callback( *cmd.args(input) )"
# * all commands have been visited and modified to be specified as
# "def xyz( .., *args)"
# and do the right thing if called with wrong number of arguments.
# Many command's regex's got simpler.
# * fixed some error messages to be more readable
# * extended the documentation of x, y scaling of plots ("xyscale")
# * clear error message if adressing an invalid panel in a plot
# * the 'new plot' specification was widened: can now use comma-separated
# list of axes to set the new plot value in one go: "new p,sb true"
# * the "cd" command now does '~' expansion at the beginning of the path
# * fixed bug in pgplot device name generation in case details were left out
#
# Revision 1.61 2015-09-16 14:56:26 jive_cc
# HV: * Feature request by Lorant to be able to show currently defined variables
#
# Revision 1.60 2015-09-02 12:48:53 jive_cc
# HV: * go to absolute page number is now also 1-based, to be consistent with
# page numbering as displayed on the plot page(s)
#
# Revision 1.59 2015-09-02 07:49:40 jive_cc
# HV: * plots on screen are now black-on-white too
# * helptext for 'ckey' was not linked to full, extented, helpfile text
#
# Revision 1.58 2015-07-08 07:56:54 jive_cc
# HV: * added integration time as global symbol in time selection 't_int'
# * updated and corrected "time" documentation
#
# Revision 1.57 2015-07-07 10:29:29 jive_cc
# HV: * opening a different MS clears the selection, as it should've been from
# the start
#
# Revision 1.56 2015-06-26 12:46:11 jive_cc
# HV: * let os.system() handle "ls" command
#
# Revision 1.55 2015-04-08 14:35:24 jive_cc
# HV: * prevent memory usage explosion
#
# Revision 1.54 2015-03-26 20:10:08 jive_cc
# HV: * flatfilter() sorts the data sets by increasing x-value. This is
# very important when executing data set expressions ("load", "store")
# * help text for "load" and "store" now actually refers to the full text,
# not the one-line summary.
# * add support for the "ckey" command - programmable data set colouring
# is now possible. data sets can be assigned explicit colours and/or
# based on a diferent set of attribute values than the default.
#
# Revision 1.53 2015-03-11 13:54:20 jive_cc
# HV: * when loading/storing an expression producing plots
# any installed post-processing is also run
# * dataset expression parser changes:
# - loose the '$' for variable dereferencing; the
# name resolving has become smarter
# - prepare to support calling functions-with-arguments
# a name followed by function-call operator '(' <args> ')'
#
# Revision 1.52 2015-02-02 08:55:21 jive_cc
# HV: * support for storing/loading plots, potentially manipulating them
# via arbitrary arithmetic expressions
# * helpfile layout improved
#
# Revision 1.51 2015-01-09 14:27:57 jive_cc
# HV: * fixed copy-paste error in weight-thresholded quantity-versus-time fn
# * sped up SOLINT processing by factor >= 2
# * output of ".... took XXXs" consistentified & beautified
# * removed "Need to convert ..." output; the predicted expected runtime
# was usually very wrong anyway.
#
# Revision 1.50 2015-01-09 00:02:27 jive_cc
# HV: * support for 'solint' - accumulate data in time bins of size 'solint'
# now also in "xxx vs time" plots. i.e. can be used to bring down data
# volume by e.g. averaging down to an arbitrary amount of seconds.
# * solint can now be more flexibly be set using d(ays), h(ours),
# m(inutes) and/or s(econds). Note that in the previous versions a
# unitless specification was acceptable, in this one no more.
#
# Revision 1.49 2015-01-06 13:53:25 jive_cc
# HV: * improved error handling when trying to "cd" whilst in a closed
# plot environment
#
# Revision 1.48 2015-01-06 12:09:53 jive_cc
# HV: * fixed lost precision issue in generating TIME part of TaQL qry when
# selecting data from scan(s)
#
# Revision 1.47 2014-12-19 14:25:35 jive_cc
# HV: * cleaned up "mark" command a bit; now more consistent
# * index now must have ":" to separate it from the expression
# * edited built-in help to reflect this syntax change
#
# Revision 1.46 2014-12-19 11:24:06 jive_cc
# HV: * Ability to close current plot device. Note that your environment goes with it ...
#
# Revision 1.45 2014-12-19 10:33:31 jive_cc
# HV: * can now load post-processing code using absolute path
#
# Revision 1.44 2014-12-19 10:08:29 jive_cc
# HV: * Plotting can now safely be interrupted using ^C, w/o exiting the whole program
#
# Revision 1.43 2014-12-11 21:25:03 jive_cc
# HV: * define extra colours, if the device supports more colours. we now have
# 48 colours available.
#
# Revision 1.42 2014-12-09 16:37:51 jive_cc
# HV: * Added possibility of adding post-processing of data sets -
# can load/run MODULE.FUNCTION() to operate on the generated datasets.
# The function can add extra drawfunctions to the data set such that
# extra things can be drawn or information can be printed to stdout.
# * The page legend code has been modified such that the space below the
# plot is used more efficiently, taking into account the size of the
# longest data set label, so the layout is more dynamic.
#
# Revision 1.41 2014-11-28 14:31:08 jive_cc
# HV: * fixed two locations where "self.dirty" was not handled properly
#
# Revision 1.40 2014-09-02 16:20:09 jive_cc
# HV: * fix crash/untimely demise if trying to plot to a file which failed to
# open
# * support for setting line width, point size and marker size for those
# respective drawing elements (and querying them, ofc).
#
# Revision 1.39 2014-08-08 15:38:41 jive_cc
# HV: * Added 'scripted' to command.py - a command-source for the commandline
# object, assuming the object generates commands. Can be generator or
# iterable.
# * jplotter gets '-h' and '-v' and '-d' command line flags
# * 'jcli' is now called 'run_plotter' and takes a commandline instance
# as parameter; the caller determines where the commands for the plotter
# are read from
# * modifications to 'jcli/run_plotter' to properly support plotting to
# file directly ... this is a bit of a hidden trick; the name of
# the environment is kept the same but the pgplot device in it is
# changed to a file. Nasty, but it works. PGPLOT only allows one
# file to be open at any point in time *sigh*.
# * support for "/cps" or "/vcps" ending in the postscript file names
# for 'save' and 'file' commands
# * added first version of pythonified standardplots!!!!!!!!!!
# Finally!!! After having been on the RnDM agenda for YEARS, the
# moment is finally there! Yay!
#
# Revision 1.38 2014-08-07 10:15:46 jive_cc
# HV: * Prevent NaN being present in the data breaking the plots
#
# Revision 1.37 2014-05-21 18:12:59 jive_cc
# HV: * Make failure to open PGPLOT device not fatal for the whole
# plotting app ...
#
# Revision 1.36 2014-05-15 20:58:21 jive_cc
# *** empty log message ***
#
# Revision 1.35 2014-05-15 09:56:09 jive_cc
# HV: * Moar beautyfication
#
# Revision 1.34 2014-05-15 08:27:27 jive_cc
# HV: * Add 'new plot' and 'solint' settings to output of the 'pp' command
# to display the full set of plot properties
#
# Revision 1.33 2014-05-15 08:19:43 jive_cc
# HV: * Beautified output of plot properties and selection / everything
# now lines up nicely when printed using 'sl' or 'pp'
#
# Revision 1.32 2014-05-14 17:35:14 jive_cc
# HV: * if weight threshold is applied this is annotated in the plot
# * the plotiterators have two implementations now, one with weight
# thresholding and one without. Until I find a method that is
# equally fast with/without weight masking
#
# Revision 1.31 2014-05-14 17:02:01 jive_cc
# HV: * Weight thresholding implemented - but maybe I'll double the code
# to two different functions, one with weight thresholding and one
# without because weight thresholding is sloooooow
#
# Revision 1.30 2014-05-12 21:22:41 jive_cc
# HV: * multiple plot environments now remember (and switch to) their current
# working directory
#
# Revision 1.29 2014-05-11 11:01:38 jive_cc
# HV: * Initial support for multiple independant plots/windows/files; each
# device has their own MS + selection + plot type. Not everything works
# perfectly yet.
#
# Revision 1.28 2014-05-06 14:32:12 jive_cc
# HV: * Scan selection accepts a number of scan numbers and/or ranges and
# rewrites it into a query
#
# Revision 1.27 2014-05-06 14:20:39 jive_cc
# HV: * Added marking capability
#
# Revision 1.26 2014-04-25 15:15:46 jive_cc
# HV: * support drawing of lines, points or both
#
# Revision 1.25 2014-04-25 12:22:29 jive_cc
# HV: * deal with lag data correctly (number of channels/number of lags)
# * there was a big problem in the polarization labelling which is now fixed
#
# Revision 1.24 2014-04-24 20:13:15 jive_cc
# HV: * 'scan' and 'time' selection now use the expression parsers rather than
# the regex-based implementation
# * trigger use of persistent macros in CommanLine object
#
# Revision 1.23 2014-04-23 14:17:46 jive_cc
# HV: * Saving this old version which deals with "scans" - to be replaced with
# something far more powerful
#
# Revision 1.22 2014-04-15 12:23:11 jive_cc
# HV: * deleted C++ makePlots method
#
# Revision 1.21 2014-04-15 12:21:56 jive_cc
# HV: * pagelabel is now centralized computed in the base class
# we now have maximum consistency between all plots
#
# Revision 1.20 2014-04-15 07:53:16 jive_cc
# HV: * time averaging now supports 'solint' = None => average all data in
# each time-range selection bin
#
# Revision 1.19 2014-04-14 22:08:01 jive_cc
# HV: * add support for accessing scan properties in time selection
#
# Revision 1.18 2014-04-14 21:04:43 jive_cc
# HV: * Information common to all plot- or data set labels is now stripped
# and displayed in the plot heading i.s.o in the plot- or data set label
#
# Revision 1.17 2014-04-14 14:46:05 jive_cc
# HV: * Uses pycasa.so for table data access waiting for pyrap to be fixed
# * added "indexr" + scan-based selection option
#
# Revision 1.16 2014-04-10 21:14:40 jive_cc
# HV: * I fell for the age-old Python trick where a default argument is
# initialized statically - all data sets were integrating into the
# the same arrays! Nice!
# * Fixed other efficiency measures: with time averaging data already
# IS in numarray so no conversion needs to be done
# * more improvements
#
# Revision 1.15 2014-04-08 22:41:11 jive_cc
# HV: Finally! This might be release 0.1!
# * python based plot iteration now has tolerable speed
# (need to test on 8M row MS though)
# * added quite a few plot types, simplified plotters
# (plotiterators need a round of moving common functionality
# into base class)
# * added generic X/Y plotter
#
# Revision 1.14 2014-04-02 17:55:30 jive_cc
# HV: * another savegame, this time with basic plotiteration done in Python
#
# Revision 1.13 2013-12-12 14:10:15 jive_cc
# HV: * another savegame. Now going with pythonic based plotiterator,
# built around ms2util.reducems
#
# Revision 1.12 2013-09-03 17:34:30 jive_cc
# HV: * Amazing! All plot types do work now. Can save plots to file.
# Still some details left to do obviously ...
#
# Revision 1.11 2013-08-20 18:23:50 jive_cc
# HV: * Another savegame
# Got plotting to work! We have stuff on Screen (tm)!
# Including the fance standardplot labels.
# Only three plot types supported yet but the bulk of the work should
# have been done I think. Then again, there's still a ton of work
# to do. But good progress!
#
# Revision 1.10 2013-06-19 12:28:44 jive_cc
# HV: * making another savegame
#
# Revision 1.9 2013-03-31 17:17:56 jive_cc
# HV: * another savegame
#
# Revision 1.8 2013-03-11 18:23:39 jive_cc
# HV: * freqsel now works properly (yay!)
# * updated source selection to be consistent with baseline selection,
# i.e.:
# selector = expr | -expr # -expr = remove from selection
# expr = name | !name # !name = negative match on name
# name = regex
#
# Revision 1.7 2013-03-10 14:33:36 jive_cc
# HV: * freqsel working (first version)
#
# Revision 1.6 2013-03-09 16:59:07 jive_cc
# HV: * another savegame
#
# Revision 1.5 2013-03-08 21:30:08 jive_cc
# HV: * making a savegame
#
# Revision 1.4 2013-02-19 16:53:29 jive_cc
# HV: * About time to commit - make sure all edits are safeguarded.
# Making good progress. baselineselection, sourceselection and
# timeselection working
#
# Revision 1.3 2013-02-11 09:40:33 jive_cc
# HV: * saving work done so far
# - almost all mapping-functionality is in place
# - some of the API functions are starting to appear
#
# Revision 1.2 2013-01-29 12:23:45 jive_cc
# HV: * time to commit - added some more basic stuff
#
from __future__ import print_function
from six import iteritems
import copy, re, math, operator, itertools, plotiterator, ppgplot, datetime, os, subprocess, numpy, parsers, imp, time
import jenums, selection, ms2mappings, plots, ms2util, hvutil, pyrap.quanta, sys, pydoc, collections, gencolors, functools
from functional import compose, const, identity, map_, filter_, drap, range_, reduce, partial, GetA
if '-d' in sys.argv:
print("PPGPLOT=",repr(ppgplot))
# Monkeypatch ppgplot if it don't seem to have pgqwin()
if not hasattr(ppgplot, 'pgqwin'):
# the associative array from device id to window
pg_windows = dict()
# need to wrap pgswin to remember which window set for which device
old_pgswin = ppgplot.pgswin
def pgswin_new(w0, w1, w2, w3):
pg_windows[ ppgplot.pgqid() ] = [w0, w1, w2, w3]
return old_pgswin(w0, w1, w2, w3)
def pgqwin():
return pg_windows.get(ppgplot.pgqid(), None)
ppgplot.pgswin = pgswin_new
ppgplot.pgqwin = pgqwin
CP = copy.deepcopy
NOW = time.time
AVG = jenums.Averaging
FLAG = jenums.Flagstuff
SYMBOL = jenums.Symbol
pwidth = lambda p, w: "{0:{1}}".format(p, w)
ppfx = lambda p: pwidth(p, 10)
pplt = lambda p: pwidth(p, 25)
prng = lambda p: pwidth(p, 14)
class jplotter:
def __init__(self, ms=None, **kwargs):
# set construction defaults
d = {'ms': ms, 'unique': True}
# update with user input
d.update( kwargs )
# wether to show ALL meta data (selectables) 'unique==False')
# or only show selectables that are actually used in the main
# table of a MS ('unique==True')
self.unique = d['unique']
# Our data members
self.reset()
# finally, see if we were constructed with a ms argument
if d['ms']:
self.ms(d['ms'])
def ms(self, *args):
if args:
rxYesNo = re.compile(r"^(?P<yes>t(rue)?)|f(alse)?$", re.I)
# provide transformation(s) for options on the command line
def as_bool(x):
mo = rxYesNo.match(x)
if mo:
return bool( mo.group('yes') )
raise RuntimeError("{0} is not a valid boolean expression".format(x))
(arguments, options) = hvutil.split_optarg(*args, **{'unique':as_bool, 'readflags':as_bool})
if len(arguments)>1:
raise RuntimeError("ms() takes only one or no parameters")
# attempt to open the measurement set
try:
# Update our defaults with what the user specified on the command line
self.mappings.load(arguments[0], **dict({'unique': self.unique}, **options))
self.msname = arguments[0]
self.scanlist = []
self.dirty = True
self.selection = selection.selection()
self.readFlags = options.get('readflags', True)
except Exception as E:
print(E)
if self.msname:
m = self.mappings
print("ms: Current MS is '{0}' containing {1} rows of {2} data for {3} [{4}]".format( \
self.msname, m.numRows, m.domain.domain, m.project, m.domain.column))
else:
print("ms: no MS opened yet")
return True
def uniqueMetaData(self, *args):
if args:
if len(args)>1:
raise RuntimeError("This command supports only one argument")
if args[0] in "tT":
self.unique = True
elif args[0] in "fF":
self.unique = False
else:
self.unique = bool(int(args[0]))
print("unique meta data: ",self.unique)
def haveMS(self):
return self.msname
def listScans(self, *args):
pfx = "listScans:"
if not self.msname:
print(pfx,"No MS loaded yet")
return None
if not self.scanlist:
print(pfx, "No scans (yet) - have you run the 'indexr' task?")
return None
# print list of scan id's + text
lines = "\n".join( map(str, self.scanlist) )
if args and '-p' in args:
pydoc.pager( lines )
else:
print(lines)
def listBaselines(self):
pfx = "listBaselines:"
if not self.msname:
print(pfx,"No MS loaded yet")
return None
mk_output(prng(pfx), self.mappings.baselineMap.baselineNames(), 80)
def listFreqs(self):
if not self.msname:
print("listFreqs: No MS loaded yet")
return
pmap = self.mappings.polarizationMap
smap = self.mappings.spectralMap
pfx = 'listFreqs:'
# deal with the frequency setups
for frqid in smap.freqIds():
print("{0} FREQID={1} [{2}]".format(prng(pfx), frqid, smap.freqGroupName(frqid)))
# Loop over the subbands
for (sbidx, sb) in smap.subbandsOfFREQ(frqid):
print("{0} SB{1:2d}:".format(prng(pfx), sbidx),sb.__str__(polarizationMap=self.mappings.polarizationMap))
def freqIDs(self):
if not self.msname:
return []
else:
return self.spectralMap.freqIds()
def nSubband(self, frqid):
if not self.msname:
return 0
else:
return self.spectralMap.spectralWindows(frqid)
def nPolarizations(self, fid):
if not self.msname:
return 0
else:
return len(self.polarizationMap.getCorrelationTypes(fid))
def Polarizations(self, pid):
if not self.msname:
return []
else:
return self.polarizationMap.getPolarizations(pid)
def listSources(self):
pfx = "listSources:"
if not self.msname:
print(pfx, "No MS loaded yet")
return None
mk_output(prng(pfx+" "), self.mappings.fieldMap.getFields(), 80)
def listAntennas(self):
pfx = "listAntennas:"
if not self.msname:
print(pfx,"No MS loaded yet")
return None
# baselineMap.antennas() = [ (id, name), ... ]
mk_output(prng(pfx+" "), map_("{0[0]} ({0[1]: >2}) ".format,
self.mappings.baselineMap.antennas()), 80)
def listTimeRange(self):
pfx = "listTimeRange:"
if not self.msname:
print(pfx,"No MS loaded yet")
return None
# call upon the mighty casa quanta to produce
# human readable format
tr = self.mappings.timeRange
print(prng(pfx),ms2util.as_time(tr.start),"->",ms2util.as_time(tr.end)," dT:"," ".join(map("{0:.3f}".format, tr.inttm)))
def dataDomain(self):
return self.mappings.domain.domain
def indexr(self):
pfx = "indexr:"
if not self.msname:
print(pfx,"No MS loaded yet")
return []
# need to remember old ms name
lmsvar = 'indexr_lastms'
if not hasattr(self, lmsvar):
setattr(self, lmsvar, None)
if not self.scanlist or self.indexr_lastms!=self.msname:
# need to recompute scan list.
# From indexr() we only get field_ids in the scan object
# so we immediately transform them into field names
print("Running indexr. This may take some time.")
unmapFLD = self.mappings.fieldMap.field
def unmapFLDfn(x):
x.field = unmapFLD(x.field_id)
return x
self.scanlist = map_(unmapFLDfn, ms2util.indexr(self.msname))
# and store the current ms
self.indexr_lastms = CP(self.msname)
print("indexr: found ",len(self.scanlist)," scans. (use 'listr' to inspect)")
return self.scanlist
## display or select time-range + source via scan
_scanTaQL = "(TIME>={0.start:.7f} AND TIME<={0.end:.7f} AND FIELD_ID={0.field_id} AND ARRAY_ID={0.array_id} AND SCAN_NUMBER={0.scan_number})".format
def scans(self, *args):
pfx = "scan:"
errf = hvutil.mkerrf("{0} ".format(pfx, self.msname))
if not self.msname:
return errf("No MS loaded yet")
# shorthand to selection object
sel_ = self.selection
map_ = self.mappings
# were we called with argument(s)?
if args:
if not self.scanlist:
return errf("No scanlist yet - please run 'indexr' first")
oldscanSel = CP(sel_.scanSel)
# if args[0] looks like a range of scan numbers, then rewrite the argument
qry = args[0]
rxIrange = re.compile(r"^\d+(-\d+(:\d+)?)?(\s+\d+(-\d+(:\d+)?)?)*$")
if rxIrange.match(qry):
qry = "start to end where scan_number in [{0}]".format(",".join(qry.split()))
# Get the modifier function and scan-selection function
tr = map_.timeRange
(mod_f, filter_f) = parsers.parse_scan(qry, start=tr.start, end=tr.end, length=tr.end-tr.start, mid=(tr.start+tr.end)/2.0)
# We create copies of the selected scans and modify them (such that the original definitions
# remain unaltered). We mainly copy all attributes for display purposes. Behind the scenes it's
# just a time range selection but that doesn't read too well on the screen
def copy_and_modify(acc, scan):
nScan = CP(scan)
(nStart, nEnd) = mod_f(scan)
nScan.start = nStart
nScan.end = nEnd
acc.append(nScan)
return acc
# Only override scan selection if any scans were selected
scanSel = reduce(copy_and_modify, filter_f(self.scanlist), [])
if not scanSel:
return errf("Your selection criteria did not match any scans")
sel_.scanSel = scanSel
self.dirty = self.dirty or (sel_.scanSel!=oldscanSel)
# Construct the TaQL from the scan selection
if self.dirty:
# clear out time/source selection
sel_.sources = []
sel_.timeRange = []
sel_.sourcesTaql = None
# replace time range taql only if there are any scans selected
scanTaQL = " OR ".join(map(jplotter._scanTaQL, sel_.scanSel))
sel_.timeRangeTaql = "("+scanTaQL+")" if scanTaQL else None
# and we must construct the timerange list.
# also compress the time ranges; if there are overlapping regions, join them
def reductor(acc, xxx_todo_changeme4):
# if current start time <= previous end-time we has an overlaps
(s, e) = xxx_todo_changeme4
if s<= acc[-1][1]:
acc[-1] = (acc[-1][0], e)
else:
acc.append( (s,e) )
return acc
# transform selected scans into list of (start,end) tuples and sort them ascending by start time
sel_.timeRange = sorted(map(operator.attrgetter('start', 'end'), sel_.scanSel), key=operator.itemgetter(0))
# Now compress them
if sel_.timeRange:
sel_.timeRange = reduce(reductor, sel_.timeRange[1:], [sel_.timeRange[0]])
# Display selected scans
if sel_.scanSel:
print("\n".join(map(str, sel_.scanSel)))
else:
print("No scans selected{0}".format( "" if self.scanlist else " (and could not have; 'indexr' must be run first)" ))
##
## Here come interesting API methods
##
## channels() => show current selection of channels
## channels(str) => select new set of channels"
def channels(self, *args):
pfx = "channels:"
errf = hvutil.mkerrf("{0} ".format(pfx, self.msname))
if not self.msname:
return errf("No MS loaded yet")
# either no args or a single string
if args:
sel_ = self.selection
spmap_ = self.mappings.spectralMap
# ah, let's see what the usr gave us.
# let's see if there's a freqid selected.
# If there's only one freqid in the MS we default to
# that one. se scream if we're using a relative
# selection like 'mid', 'all', 'first', 'last'
if args[0]=="none":
sel_.chanSel = None
else:
# get the current datadescription id's and verify the
# number of channels is equal for all
nchan = None
if sel_.ddSelection:
# ddSelection is [(fq, sb, pol, [idx, ..]), ... ]
nch = set(map(lambda f_s_p_l: spmap_.numchanOfFREQ_SB(f_s_p_l[0], f_s_p_l[1]), sel_.ddSelection))
if len(nch)==1:
nchan = list(nch)[0]
else:
# no freq selection yet, see if all the spectral windows in all the
# freqids have the same number of channels. If they do we can easily
# use that "nch" as symbolic value
chset = reduce( \
lambda acc, fid: acc.update(map(spmap_.numchanOfSPW, spmap_.spectralWindows(fid))) or acc,\
spmap_.freqIds(), \
set())
if len(chset)==1:
# I don't know how else to extract the single element out of a set
[nchan] = list(chset)
# Rite. We may have a value for nch!
# Scream loudly if one of the 'relative' channels (other than first)
# is mentioned in the argument and we didn't find an actual value
found = lambda x : args[0].find(x)!=-1
if any([found(x) for x in ["mid","last"]]) and not nchan:
return errf("Your selection contains relative channels (mid, last) but no value for them could be deduced")
# Now we know we can safely replace strings -> values (and remove whitespace)
if nchan:
replacer = lambda x : hvutil.sub(x, [(r"\s+", r""), (re.compile(r"all"), r"first:last"), \
(r"first", r"0"), (r"mid", repr(nchan//2)), (r"last", repr(nchan-1))])
else:
replacer = lambda x : hvutil.sub(x, [(r"\s+", r"")])
expander = lambda x : hvutil.expand_string_range(replacer(x))
# Reduce and build the actual channelselection - we gather all the sub-selections (the comma separated
# entries) into a set [such that duplicates are automatically handled] and transform it into a list
# for subsequent use, which expects the chanSel to be a list
sel_.chanSel = list(reduce(lambda acc, x: acc.update(expander(x)) or acc, args, set()))
self.dirty = True
# print the selection
cstr = "No channels selected yet"
if self.selection.chanSel:
cstr = hvutil.range_repr(hvutil.find_consecutive_ranges(self.selection.chanSel))
print("{0} {1}".format(ppfx(pfx), cstr))
##
## Print or set the source selection
##
def sources(self, *args):
pfx = "sources:"
errf = hvutil.mkerrf("{0} ".format(pfx, self.msname))
if not self.msname:
return errf("No MS loaded yet")
sel_ = self.selection
map_ = self.mappings
if args:
if "none" in args:
sel_.sources = None
sel_.sourcesTaql = None
else:
# sources = select | -select
# select = expr | !expr
# expr = shell like regex, meaning
# "*" for wildcard [0 or more characters]
# "?" for one character
#
# take [(sourcename, flag),..] and [selectors(),...]
# and map each selector over that list of (source,flag) tuples.
# the selector function will return a new list of (source,flag) tuples
# with the flags updated according to its selection result.
# Return the list of sources for which the flag is true
#
selector = lambda tuplst, selectors: \
[src for (src, flag) in reduce(lambda acc, sel: map(sel, acc), selectors, tuplst) if flag]
def mkselector(src):
srcs = copy.deepcopy(src)
# 0) Check if it was an expression with an explicit add/subtract
add = True
mo = re.match("^\s*(?P<add>[-+])(?P<expr>.*)", srcs)
if mo:
add = (mo.group('add')=='+')
srcs = mo.group('expr')
# a) remove whitespace and replace multiple wildcards by a single one
srcs = hvutil.sub(srcs, [(r"\s+", r""), (r"\*+", r"*"), (r"^all$", r"*")])
# b) was a negation given? If so, strip the negation character but remember it
neg = re.match("^!", srcs)
if neg:
srcs = srcs[1:]
# c) escape re special characters: "+" => "\+", "." => "\."
# so they won't be interpreted *as* regex characters - they're
# wont to appear in source names
# Also replace the question mark by "." - do this _after_ the
# literal dots have been escaped ...
srcs = hvutil.sub(srcs, [(r"\+", r"\+"), (r"\.", r"\."), (r"\?", r".")])
# d) replace wildcards (*) by (.*)
srcs = hvutil.sub(srcs, [(r"\*", r".*")])
# Now we can make a regex
rx = re.compile(r"^"+srcs+"$", re.I)
return lambda src_flag: (src_flag[0], (src_flag[1] if neg else add) if rx.match(src_flag[0]) else (add if neg else src_flag[1]))
# now build the list of selectors, based on comma-separated source selections
# and run all of them against the sourcelist
sel_.sources = selector([(x, False) for x in map_.fieldMap.getFields()], map(mkselector, args))
# set the TaQL accordingly - if necessary
if len(sel_.sources)==len(map_.fieldMap.getFields()):
# all sources selected - no need for TaQL
sel_.sourcesTaql = ""
else:
sel_.sourcesTaql = "(FIELD_ID IN {0})".format( map(map_.fieldMap.unfield, sel_.sources) )
self.dirty = True
# new source selection overrides scan selection
sel_.scanSel = []
sstr = ["No sources selected yet"]
if sel_.sources:
sstr = sel_.sources
mk_output(ppfx(pfx), sstr, 80)
##
## print/edit the baseline selection
##
def baselines(self, *args):
pfx = "baselines:"
errf = hvutil.mkerrf("{0} ".format(pfx, self.msname))
if not self.msname:
return errf("No MS loaded yet")
sel_ = self.selection
if args:
# remember current TaQL for the baselines so we can detect if the
# selection changed
oldTaql = copy.deepcopy(sel_.baselinesTaql)
pr = parsers.parse_baseline_expr(args[0], self.mappings.baselineMap)
sel_.baselines = map_(GetA('BL'), pr.baselines)
sel_.baselinesTaql = copy.deepcopy(pr.taql)
self.dirty = self.dirty or oldTaql!=sel_.baselinesTaql
blstr = ["No baselines selected yet"]
if sel_.baselines:
blstr = sel_.baselines
mk_output(ppfx(pfx), blstr, 80)
##
## print/set the timerange(s)
##
def times(self, *args):
pfx = "time:"
errf = hvutil.mkerrf("{0} ".format(pfx, str(self.msname)))
if not self.msname:
return errf("No MS loaded yet")
if len(args)>1:
raise RuntimeError("Please call this method with zero or one argument")
sel_ = self.selection
if args:
if "none" in args:
sel_.timeRange = None
sel_.timeRangeTaql = None
else:
tr = self.mappings.timeRange
# Parse the time ranges
l = parsers.parse_time_expr(args[0], start=tr.start, end=tr.end, length=tr.end-tr.start, \
mid=(tr.start+tr.end)/2.0, t_int=tr.inttm[0])
# Sort by start time & compress; overlapping time ranges get merged
sel_.timeRange = sorted(l, key=operator.itemgetter(0))
def reductor(acc, s_e):
# if current start time <= previous end-time we has an overlaps
if s_e[0]<= acc[-1][1]:
acc[-1] = (acc[-1][0], s_e[1])
else:
acc.append( s_e )
return acc
if sel_.timeRange:
sel_.selectTimeRange( reduce(reductor, sel_.timeRange[1:], [sel_.timeRange[0]]) )
self.dirty = True
sel_.scanSel = []
# and show the current timeselection
if sel_.timeRange:
mk_output(ppfx(pfx), map(lambda s_e: "{0} -> {1}".format(ms2util.as_time(s_e[0]), ms2util.as_time(s_e[1])), sel_.timeRange), 80)
else:
print("{0} No timerange(s) selected yet".format(ppfx(pfx)))
##
## Support frequency selection
##
def freqsel(self, *args):
pfx = "freqsel:"
errf = hvutil.mkerrf("{0} ".format(pfx, self.msname))
if not self.msname:
return errf("No MS loaded yet")
sel_ = self.selection
spMap = self.mappings.spectralMap
pMap = self.mappings.polarizationMap
# [<freqidsel>/]<subbandselection>[/<polselection>]
#
# Where we will accept the following for the separate entries:
#
# freqidsel: * OR
# ranges (eg: '1:5[:2]') AND/OR separate entries (commaseparated)
#
# Note: freqidsel may be omitted if and only if there is
# only one freqid!
#
# subbandselection: same as freqidsel
#
# polselection: It is optional. If no polsel, the first polid is used and
# all combis are used.
# Syntax:
# [polid:]pols
# polid = if desired, the polarization selection
# can be prefixed with a specific polarization
# ID to draw the selected polarizations from.
# If not specified, the first polarization id
# that yields positive match(es) on _all_
# specified pols is used
# pols = comma separated list of:
# [rl]{2}, *[lr], [lr]*, X, P, A
# X = select crosspols
# P = select parallel polarizations
# or just
# *
#
# Examples:
# */P selects all parallel hand polarization combinations from
# all subbands from the default freqid. Only allowed iff
# there is only one freqid! Note: if the subband(s) are
# correlated with multiple polarization ids then the first
# polarization id with parallel polarizations will be
# selected.
# */* selects all polarizations from the first polarization ID
# for all subbands. This is allowed only iff there is one
# freqid!
# */1:P selects all parallel hand polarization combinations from
# polarization id #1 for all subbands for the default freqid.
# Only iff there is only one freqid. Each subband must
# be (at least) correlated with polarization id #1
# 1,3,5/2:X selects the crosspols from polarization ID 2 for
# subbands 1, 3 and 5. Also only allowed iff there is
# only one freqid
# 2/0:3/r* selects polarizations RR and/or RL (if they exist)
# from the first polarization ID that has either or both
# for subbands 0,1,2,3 from freqid #2
# 0/1:4/0:rr,ll explicitly select (require) from polarization ID #0
# RR and LL for subbands 1,2,3,4 from freqid #0
#
# We need to end up with a list of
# [(DATA_DESC_ID, [CORR_TYPE,...]), ...]
# or, in English:
# for each DATA_DESCRIPTION_ID ("DDId") we select we must record which products
# out of the polarization products associated with that DDId are selected.
# regexes
#rxPol = re.compile(r"^((?P<polid>\d+):)?(?P<pexpr>\*|(([RrlL]{2}|\*[RrLl]|[RrLl]\*|[XP])(,([RrLl]{2}|\*[RrLl]|[RrLl]\*|[XP]))*))$")
rxPol = re.compile(r"^((?P<polid>\d+):)?(?P<pexpr>\*|(([rl]{2}|\*[rl]|[rl]\*|[XP])(,([rl]{2}|\*[rl]|[rl]\*|[XP]))*))$", re.I)
rxRng = re.compile(r"^\*|(\d+|\d+:\d+)(,(\d+|\d+:\d+))*$")
if args:
#if re.match("^\s*$", args[0]):
# raise RuntimeError, "Empty frequency selection string?"
if "none" in args:
sel_.ddSelection = None
sel_.ddSelectionTaql = None
else:
# eventually we'll end up with a list of selected datadescription IDs
# each selection entry will add the DDids it selects to the current set
def mkselector(arg):
# the three potential bits in the selection
fqs = sbs = pols = None