From bab669d33aa8de0ea999b1117b61577a892c9263 Mon Sep 17 00:00:00 2001 From: TheSkyentist Date: Thu, 7 Mar 2024 20:40:00 +0100 Subject: [PATCH 1/9] Fixed DQ bug for JWST, correct bits now set --- grizli/prep.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grizli/prep.py b/grizli/prep.py index 4df134c4..10ec0875 100644 --- a/grizli/prep.py +++ b/grizli/prep.py @@ -6475,7 +6475,7 @@ def visit_grism_sky(grism={}, apply=True, column_average=True, verbose=True, ext if isACS: bits = 64+32 elif isJWST: - bits = 1+4+6+32768 #+16777200+1049600+26232800+9438210+9438220 + bits = 2+4+32768 #+16777200+1049600+26232800+9438210+9438220 else: bits = 576 From 47c989921c8a859df45bd3cfc4ce96a650b23a31 Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Mon, 18 Mar 2024 16:51:10 +0100 Subject: [PATCH 2/9] parse int cores --- grizli/aws/recalibrate.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/grizli/aws/recalibrate.py b/grizli/aws/recalibrate.py index efb2c422..30af72f8 100644 --- a/grizli/aws/recalibrate.py +++ b/grizli/aws/recalibrate.py @@ -227,11 +227,16 @@ def do_recalibrate(rate_file='jw06541001001_03101_00001_nrs1_rate.fits', cores=' output_path, base_file = os.path.split(rate_file) mastquery.utils.download_from_mast([base_file.replace('_rate.fits','_uncal.fits')]) - + + try: + ncores = int(cores) + except ValueError: + ncores = cores + steps = { "jump": { "save_results": True, - "maximum_cores": cores + "maximum_cores": ncores }, "ramp_fit": { "skip": True, @@ -248,9 +253,9 @@ def do_recalibrate(rate_file='jw06541001001_03101_00001_nrs1_rate.fits', cores=' after_jumps=int(after_jumps), save_results=True, suffix="snowblind") - + rate, rateints = RampFitStep.call(base_file.replace('_rate','_snowblind'), - maximum_cores=cores) + maximum_cores=ncores) rate = GainScaleStep.call(rate) From 48cbda0bb9837a5842b01b3c6fc127d9681430d9 Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Thu, 11 Apr 2024 20:04:27 +0200 Subject: [PATCH 3/9] enable LS_DR10 reference catalog --- grizli/catalog.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/grizli/catalog.py b/grizli/catalog.py index c154207d..a2ce8caf 100644 --- a/grizli/catalog.py +++ b/grizli/catalog.py @@ -1128,7 +1128,7 @@ def get_radec_catalog(ra=0., dec=0., radius=3., product='cat', verbose=True, ref Order in which to query reference catalogs. Options are 'GAIA', 'PS1' (STScI PanSTARRS), 'SDSS', 'WISE', 'NSC' (NOAO Source Catalog), 'DES' (Dark Energy Survey DR1), 'Hubble' (Hubble Source Catalog v3), - 'LS_DR9' (LegacySurveys DR9). + 'LS_DR9', 'LS_DR10' (LegacySurveys DR9, DR10). Returns ------- @@ -1139,6 +1139,10 @@ def get_radec_catalog(ra=0., dec=0., radius=3., product='cat', verbose=True, ref Provenance of the `radec` list. """ + query_kwargs = {'LS_DR9': {'db':'ls_dr9.tractor'}, + 'LS_DR10': {'db':'ls_dr10.tractor'}, + } + query_functions = {'SDSS': get_sdss_catalog, 'GAIA_TAP': get_gaia_DR2_catalog, 'PS1': get_panstarrs_catalog, @@ -1151,7 +1155,9 @@ def get_radec_catalog(ra=0., dec=0., radius=3., product='cat', verbose=True, ref 'Hubble': get_hubble_source_catalog, 'Skymapper': get_skymapper_catalog, 'VEXAS': get_vexas_catalog, - 'LS_DR9': get_legacysurveys_catalog} + 'LS_DR9': get_legacysurveys_catalog, + 'LS_DR10': get_legacysurveys_catalog, + } # Try queries has_catalog = False @@ -1176,8 +1182,13 @@ def get_radec_catalog(ra=0., dec=0., radius=3., product='cat', verbose=True, ref ref_cat = query_functions[ref_src](ra=ra, dec=dec, radius=radius, use_mirror=True) else: + if ref_src in query_kwargs: + _kws = query_kwargs[ref_src] + else: + _kws = {} + ref_cat = query_functions[ref_src](ra=ra, dec=dec, - radius=radius) + radius=radius, **_kws) # # # ref_cat = query_functions[ref_src](ra=ra, dec=dec, # radius=radius) From 785b38613280451d11609ba78f687afa44ce9d39 Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Fri, 19 Apr 2024 17:11:13 +0200 Subject: [PATCH 4/9] compute time-dependent photometric correction for MIRI --- grizli/jwst_utils.py | 116 ++++++++++++++++++++++++++++++++++++++ grizli/tests/test_jwst.py | 27 +++++++++ 2 files changed, 143 insertions(+) diff --git a/grizli/jwst_utils.py b/grizli/jwst_utils.py index 43f9bee2..ff1effa5 100644 --- a/grizli/jwst_utils.py +++ b/grizli/jwst_utils.py @@ -2589,6 +2589,8 @@ def get_crds_zeropoint(instrument='NIRCAM', detector='NRCALONG', filter='F444W', Get ``photmjsr`` photometric zeropoint for a partiular JWST instrument imaging mode + To-do: add MIRI time-dependence + Parameters ---------- instrument : str @@ -3366,3 +3368,117 @@ def compute_siaf_pa_offset(c1, c2, c2_pa=202.9918, swap_coordinates=True, verbos print(f'Catalog offset: {dx[0].to(u.arcsec):5.2f} {dx[1].to(u.arcsec):5.2f} new APA: {c2_pa + dphi:.5f}') return c2_pa + dphi, dphi + + +def get_miri_photmjsr(file=None, filter='F770W', subarray='FULL', mjd=60153.23, photom_file='jwst_miri_photom_0201.fits', verbose=True): + """ + Get time-dependent MIRI photometry values + + Parameters + ---------- + file : str + Image filename + + filter : str + MIRI filter name + + subarray : str + Detector subarray used + + mjd : float + Observation epoch + + photom_file : str + CRDS ``photom`` reference file name + + verbose : bool + messaging + + Returns + ------- + photmjsr : float + Photometric scaling + + photom_corr : float + Time-dependent correction for the filter and observation epoch + + """ + import astropy.io.fits as pyfits + import jwst.datamodels + + if file is not None: + with pyfits.open(file) as im: + h = im[0].header + filter = h['FILTER'] + mjd = h['EXPSTART'] + try: + subarray = h['SUBARRAY'] + except KeyError: + pass + + try: + from jwst.photom.miri_imager import time_corr_photom + except ImportError: + # msg = 'Failed to import `jwst.photom.miri_imager` to include time-dependence' + # utils.log_comment(utils.LOGFILE, msg, verbose=verbose) + + time_corr_photom = time_corr_photom_copy + + PATH = os.path.join(os.getenv('CRDS_PATH'), 'references', 'jwst', 'miri') + local_file = os.path.join(PATH, photom_file) + remote_file = 'https://jwst-crds.stsci.edu/unchecked_get/references/jwst/' + remote_file += 'jwst_miri_photom_0201.fits' + + use_path = local_file if os.path.exists(local_file) else remote_file + + with jwst.datamodels.open(use_path) as ref: + + test = ref.phot_table['filter'] == filter + test &= ref.phot_table['subarray'] == subarray + if test.sum() == 0: + msg = f'Row not found in {photom_file} for {filter} / {subarray}' + utils.log_comment(utils.LOGFILE, msg, verbose=verbose) + + return np.nan, None + + row = np.where(test)[0][0] + photmjsr = ref.phot_table['photmjsr'][row] + + try: + photom_corr = time_corr_photom(ref.timecoeff[row], mjd) + except: + photom_corr = 0. + + + return (photmjsr, photom_corr) + + +def time_corr_photom_copy(param, t): + """ + Short Summary + -------------- + Time dependent PHOTOM function. + + The model parameters are amplitude, tau, t0. t0 is the reference day + from which the time-dependent parameters were derived. This function will return + a correction to apply to the PHOTOM value at a given MJD. + + N.B.: copied from [jwst.photom.miri_imager](https://github.com/spacetelescope/jwst/blob/master/jwst/photom/miri_imager.py#L9) + + Parameters + ---------- + param : numpy array + Set of parameters for the PHOTOM value + t : int + Modified Julian Day (MJD) of the observation + + Returns + ------- + corr: float + The time-dependent correction to the photmjsr term. + """ + + amplitude, tau, t0 = param["amplitude"], param["tau"], param["t0"] + corr = amplitude * np.exp(-(t - t0)/tau) + + return corr \ No newline at end of file diff --git a/grizli/tests/test_jwst.py b/grizli/tests/test_jwst.py index 9d2bb442..61c1836c 100644 --- a/grizli/tests/test_jwst.py +++ b/grizli/tests/test_jwst.py @@ -180,6 +180,33 @@ def test_filter_info(self): info = jwst_utils.get_jwst_filter_info(header) assert(info['name'] == 'F560W') assert(np.allclose(info['pivot'], 5.632612)) + + + def test_miri_photom(self): + + photmjsr, corr = jwst_utils.get_miri_photmjsr( + file=None, + filter='F770W', + subarray='FULL', + mjd=60153.23, + photom_file='jwst_miri_photom_0201.fits', + verbose=True, + ) + + assert np.allclose([photmjsr], 0.25890106) + assert np.allclose([corr], 0.) + + # Missing filter + photmjsr, corr = jwst_utils.get_miri_photmjsr( + file=None, + filter='xF770W', + subarray='FULL', + mjd=60153.23, + photom_file='jwst_miri_photom_0201.fits', + verbose=True, + ) + + assert np.isnan(photmjsr) class TestJWSTFittingTools: From 5a157ba63e91638af2d7dc978fcd6ad1bb2afa87 Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Fri, 19 Apr 2024 17:22:20 +0200 Subject: [PATCH 5/9] die without jwst in test --- grizli/tests/test_jwst.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/grizli/tests/test_jwst.py b/grizli/tests/test_jwst.py index 61c1836c..a47353f3 100644 --- a/grizli/tests/test_jwst.py +++ b/grizli/tests/test_jwst.py @@ -184,6 +184,11 @@ def test_filter_info(self): def test_miri_photom(self): + try: + import jwst + except ImportError: + return None + photmjsr, corr = jwst_utils.get_miri_photmjsr( file=None, filter='F770W', From 4de6348bcd569c1b037710a8fc5585153f44298a Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Fri, 19 Apr 2024 17:27:16 +0200 Subject: [PATCH 6/9] update checkout version --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 266ee9e5..24d5c92b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -35,7 +35,7 @@ jobs: python-version: ['3.10'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: From 4ec826493de0d502c24ef4f4ec9715e05320d2d2 Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Fri, 19 Apr 2024 17:29:27 +0200 Subject: [PATCH 7/9] update action versions --- .github/workflows/python-package.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 24d5c92b..8effc3d8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -17,9 +17,9 @@ jobs: python-version: [3.8, 3.9, '3.10'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Build check @@ -37,7 +37,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package @@ -54,11 +54,11 @@ jobs: python-version: [3.8, 3.9, '3.10'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package From 4751e8924a2daa74348a9c0e21a03b3167b7d3ee Mon Sep 17 00:00:00 2001 From: Gabriel Brammer Date: Fri, 19 Apr 2024 17:33:49 +0200 Subject: [PATCH 8/9] update action versions --- .github/workflows/publish-to-pypi.yml | 6 +++--- .github/workflows/python-package-with-jwst.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 22a46f12..83e7c017 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -36,14 +36,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Build sdist run: | pip install build python -m build --sdist - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: dist/*.tar.gz @@ -56,7 +56,7 @@ jobs: github.event.ref_type == 'tag' && github.ref == 'refs/heads/${{ github.event.repository.default_branch }}' steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: artifact path: dist diff --git a/.github/workflows/python-package-with-jwst.yml b/.github/workflows/python-package-with-jwst.yml index e13e36b4..afeaaf5b 100644 --- a/.github/workflows/python-package-with-jwst.yml +++ b/.github/workflows/python-package-with-jwst.yml @@ -21,9 +21,9 @@ jobs: python-version: ['3.9', '3.10'] os: [ubuntu-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install package From b2a50d8946782b9679716d98d0e7accbfe50d964 Mon Sep 17 00:00:00 2001 From: TheSkyentist Date: Fri, 8 Mar 2024 11:57:18 +0100 Subject: [PATCH 9/9] Update unset_dq_bits to mod_dq_bits --- grizli/combine.py | 2 +- grizli/jwst_utils.py | 2 +- grizli/model.py | 4 +-- grizli/prep.py | 4 +-- grizli/utils.py | 60 ++++++++------------------------------------ 5 files changed, 17 insertions(+), 55 deletions(-) diff --git a/grizli/combine.py b/grizli/combine.py index df651426..e31e5397 100644 --- a/grizli/combine.py +++ b/grizli/combine.py @@ -173,7 +173,7 @@ def combine_flt(files=[], output='exposures_cmb.fits', grow=1, print('{0:3d} {1:s} {2:6.1f} {3:6.1f} {4:10.2f}'.format(i+1, file, x0[i], y0[i], im[0].header['EXPTIME'])) - dq = utils.unset_dq_bits(im['DQ'].data, okbits=608, + dq = utils.mod_dq_bits(im['DQ'].data, okbits=608, verbose=False) wht = 1./im['ERR'].data**2 wht[(im['ERR'].data == 0) | (dq > 0) | (~np.isfinite(wht))] = 0 diff --git a/grizli/jwst_utils.py b/grizli/jwst_utils.py index ff1effa5..59b8f580 100644 --- a/grizli/jwst_utils.py +++ b/grizli/jwst_utils.py @@ -1025,7 +1025,7 @@ def exposure_oneoverf_correction(file, axis=None, thresholds=[5,4,3], erode_mask else: erode_mask = True - dq = utils.unset_dq_bits(im['DQ'].data, 4) + dq = utils.mod_dq_bits(im['DQ'].data, okbits=4) dqmask = dq == 0 mask = dqmask diff --git a/grizli/model.py b/grizli/model.py index 8471f06f..8f10dcab 100644 --- a/grizli/model.py +++ b/grizli/model.py @@ -1687,7 +1687,7 @@ def __init__(self, sci=None, err=None, dq=None, self.MW_EBV = 0. def unset_dq(self): - """Flip OK data quality bits using utils.unset_dq_bits + """Flip OK data quality bits using utils.mod_dq_bits OK bits are defined as @@ -1709,7 +1709,7 @@ def unset_dq(self): else: okbits = okbits_instrument[self.instrument] - self.data['DQ'] = utils.unset_dq_bits(self.data['DQ'], okbits=okbits) + self.data['DQ'] = utils.mod_dq_bits(self.data['DQ'], okbits=okbits) def flag_negative(self, sigma=-3): diff --git a/grizli/prep.py b/grizli/prep.py index 10ec0875..6273c00e 100644 --- a/grizli/prep.py +++ b/grizli/prep.py @@ -6481,7 +6481,7 @@ def visit_grism_sky(grism={}, apply=True, column_average=True, verbose=True, ext for i in range(Nexp): flt = pyfits.open(grism['files'][i]) - dq = utils.unset_dq_bits(flt['DQ', ext].data, okbits=bits) + dq = utils.mod_dq_bits(flt['DQ', ext].data, okbits=bits) dq_mask = dq == 0 # Data @@ -6871,7 +6871,7 @@ def fix_star_centers(root='macs1149.6+2223-rot-ca5-22-032.0-f105w', sci = images[i]['SCI'].data[sly, slx] dq = images[i]['DQ'].data[sly, slx] - dqm = dq - (dq & 2048) + dqm = utils.mod_dq_bits(dq,okbits=2048) err = images[i]['ERR'].data[sly, slx] mask = satpix[sly, slx] diff --git a/grizli/utils.py b/grizli/utils.py index 5c52a856..c4c6b822 100644 --- a/grizli/utils.py +++ b/grizli/utils.py @@ -1621,46 +1621,9 @@ def flt_to_dict(fobj, primary_keys=DEFAULT_PRIMARY_KEYS, extensions=[('SCI', i+1 return flt_dict - -def get_set_bits(value): - """ - Compute which binary bits are set for an integer - """ - - if hasattr(value, '__iter__'): - values = value - single = False - else: - values = [value] - single = True - - result = [] - - for v in values: - try: - bitstr = np.binary_repr(v)[::-1] - except: - result.append([]) - - nset = bitstr.count('1') - setbits = [] - - j = -1 - for i in range(nset): - j = bitstr.index('1', j+1) - setbits.append(j) - - result.append(setbits) - - if single: - return result[0] - else: - return result - - -def unset_dq_bits(value, okbits=32+64+512, verbose=False): +def mod_dq_bits(value, okbits=32+64+512, badbits=0, verbose=False): """ - Unset bit flags from a DQ array + Modify bit flags from a DQ array For WFC3/IR, the following DQ bits can usually be unset: @@ -1675,6 +1638,9 @@ def unset_dq_bits(value, okbits=32+64+512, verbose=False): okbits : int Bits to unset + badbits : int + Bits to set + verbose : bool Print some information @@ -1683,16 +1649,12 @@ def unset_dq_bits(value, okbits=32+64+512, verbose=False): new_value : int, `~numpy.ndarray` """ - bin_bits = np.binary_repr(okbits) - n = len(bin_bits) - for i in range(n): - if bin_bits[-(i+1)] == '1': - if verbose: - print(2**i) - value -= (value & 2**i) + if verbose: + print(f'Unset bits: {np.binary_repr(okbits)}') + print(f'Set bits: {np.binary_repr(badbits)}') - return value + return (value & ~okbits) | badbits def detect_with_photutils(sci, err=None, dq=None, seg=None, detect_thresh=2., @@ -6314,11 +6276,11 @@ def drizzle_from_visit(visit, output=None, pixfrac=1., kernel='point', dq = flt[('DQ', ext)].data & (1+1024+4096) dq |= bpdata.astype(dq.dtype) - # dq0 = unset_dq_bits(flt[('DQ', ext)].data, bits) | bpdata + # dq0 = mod_dq_bits(flt[('DQ', ext)].data, okbits=bits) | bpdata # print('xxx', (dq > 0).sum(), (dq0 > 0).sum()) else: - dq = unset_dq_bits(flt[('DQ', ext)].data, bits) | bpdata + dq = mod_dq_bits(flt[('DQ', ext)].data, okbits=bits) | bpdata wht = 1/err**2