From 3e34f7b704a0904b697d0319745681f09542d11c Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 16 Oct 2024 18:02:47 -0400 Subject: [PATCH 01/26] rearrange functions in detector_physics.py to sca.py and detector_effects.py --- config/was.yaml | 8 +- roman_imsim/__init__.py | 5 + roman_imsim/detector_effects.py | 835 ++++++++++++++++++++++++++++++++ roman_imsim/sca.py | 344 ++++++++++--- 4 files changed, 1127 insertions(+), 65 deletions(-) create mode 100644 roman_imsim/detector_effects.py diff --git a/config/was.yaml b/config/was.yaml index 8c9a1640..28e31436 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -96,7 +96,11 @@ image: nonlinearity: True ipc: True read_noise: True - sky_subtract: False + sky_subtract: True + + dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt + sca_filepath: /hpc/group/cosmology/phy-lsst/roman_sensors + save_diff: False # nobjects: 500 @@ -126,7 +130,7 @@ input: obseq_data: # file_name: /lus/grand/projects/RomanDESC/final_roman_runfiles/was/Roman_WAS_obseq_11_1_23.fits file_name: /hpc/group/cosmology/OpenUniverse2024/RomanWAS/Roman_WAS_obseq_11_1_23.fits - visit: 1 + visit: 4906 SCA: '@image.SCA' roman_psf: SCA: '@image.SCA' diff --git a/roman_imsim/__init__.py b/roman_imsim/__init__.py index 72da6e7a..a7919744 100644 --- a/roman_imsim/__init__.py +++ b/roman_imsim/__init__.py @@ -16,3 +16,8 @@ from .skycat import * from .stamp import * from .wcs import * +from .skycat import * +from .photonOps import * +from .bandpass import * +# from .detector_physics import * +from .detector_effects import * \ No newline at end of file diff --git a/roman_imsim/detector_effects.py b/roman_imsim/detector_effects.py new file mode 100644 index 00000000..ba116e17 --- /dev/null +++ b/roman_imsim/detector_effects.py @@ -0,0 +1,835 @@ +sca_number_to_file = { + 1 : 'SCA_22066_211227_v001.fits', + 2 : 'SCA_21815_211221_v001.fits', + 3 : 'SCA_21946_211225_v001.fits', + 4 : 'SCA_22073_211229_v001.fits', + 5 : 'SCA_21816_211222_v001.fits', + 6 : 'SCA_20663_211102_v001.fits', + 7 : 'SCA_22069_211228_v001.fits', + 8 : 'SCA_21641_211216_v001.fits', + 9 : 'SCA_21813_211219_v001.fits', + 10 : 'SCA_22078_211230_v001.fits', + 11 : 'SCA_21947_211226_v001.fits', + 12 : 'SCA_22077_211230_v001.fits', + 13 : 'SCA_22067_211227_v001.fits', + 14 : 'SCA_21814_211220_v001.fits', + 15 : 'SCA_21645_211228_v001.fits', + 16 : 'SCA_21643_211218_v001.fits', + 17 : 'SCA_21319_211211_v001.fits', + 18 : 'SCA_20833_211116_v001.fits', + } + +import os +import fitsio as fio +import galsim as galsim +import numpy as np +import galsim.roman as roman +from galsim.config import ParseValue +from roman_imsim.obseq import ObSeqDataLoader + +class get_pointing(object): + """ + Class to store stuff about the telescope + """ + def __init__(self, params, visit, SCA): + + self.params = params + file_name = params['input']['obseq_data']['file_name'] + obseq_data = ObSeqDataLoader(file_name, visit, SCA, logger=None) + self.filter = obseq_data.ob['filter'] + self.sca = obseq_data.ob['sca'] + self.visit = obseq_data.ob['visit'] + self.date = obseq_data.ob['date'] + self.exptime = obseq_data.ob['exptime'] + self.bpass = roman.getBandpasses()[self.filter] + self.WCS = roman.getWCS(world_pos = galsim.CelestialCoord(ra=obseq_data.ob['ra'], \ + dec=obseq_data.ob['dec']), + PA = obseq_data.ob['pa'], + date = self.date, + SCAs = self.sca, + PA_is_FPA = True + )[self.sca] + self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2),int(roman.n_pix/2))) + +class detector_effects(object): + """ + Class to simulate non-idealities and noise of roman detector images. + """ + def __init__(self, params, visit, sca, filter, logger, rng, rng_iter, sca_filepath=None): + self.sca = sca + self.filter = filter + if sca_filepath: + self.df = fio.FITS(os.path.join(sca_filepath, sca_number_to_file[self.sca])) + print('------- Using SCA files --------') + else: + self.df = None + print('------- Using simple detector model --------') + + self.params = params + self.rng = rng + self.noise = galsim.PoissonNoise(self.rng) + self.rng_np = np.random.default_rng(rng_iter) + self.pointing = get_pointing(self.params, visit, sca) + self.exptime = self.pointing.exptime + self.logger = logger + + self.force_cvz = False + if 'force_cvz' in self.params['image']['wcs']: + if self.params['image']['wcs']['force_cvz']: + self.force_cvz=True + + self.save_diff = False + if 'save_diff' in self.params['image']: + self.save_diff = bool(self.params['image']['save_diff']) + + def set_diff(self, im=None): + if self.save_diff: + self.pre = im.copy() + self.pre.write('bg.fits', dir=self.params['diff_dir']) + return + + def diff(self, msg, im=None, verbose=True): + if self.save_diff: + diff = im-self.pre + diff.write('%s_diff.fits'%msg , dir=self.params['diff_dir']) + self.pre = im.copy() + im.write('%s_cumul.fits'%msg, dir=self.params['diff_dir']) + return + + def qe(self, im): + """ + Apply the wavelength-independent relative QE to the image. + Input + im : Image + RELQE1[4096,4096] : relative QE map + """ + + im.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array + return im + + def bfe(self, im): + """ + Apply brighter-fatter effect. + Brighter fatter effect is a non-linear effect that deflects photons due to the + the eletric field built by the accumulated charges. This effect exists in both + CCD and CMOS detectors and typically percent level change in charge. + The built-in electric field by the charges in pixels tends to repulse charges + to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. + This effect can also be understood effectly as change in pixel area and pixel + boundaries. + BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change + in the detector effect charaterization file. Kernel of the total pixel area, however, + is not sufficient. Image simulation of the brighter fatter effect requires the shift + of the four pixel boundaries. Before we get better data, we solve for the boundary + shift components from the kernel of total pixel area by assumming several symmetric constraints. + Input + im : Image + BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 + """ + + nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) + bin_size = 128 + n_max = 32 + m_max = 32 + num_grids = 4 + n_sub = n_max//num_grids + m_sub = m_max//num_grids + + ##======================================================================= + ## solve boundary shfit kernel aX components + ##======================================================================= + a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 + a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 + + ##solve aR aT aL aB for each a + for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) + for m in range(m_max): + a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) + + ## assume two parity symmetries + a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. + + r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection + B = (a[2,2], a[3,2], a[2,3], a[3,3], + a[4,2], a[2,4], a[3,4], a[4,4] ) + + A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], + [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], + [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], + [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], + [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], + [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], + [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], + [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) + + + s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] + + aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s3 , -s1 , s1 , s3 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) + + + aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], + [ -s7 , -s6 , -s4 , -s6 , -s7 ], + [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], + [ r*s6 , s5 , s2 , s5 , r*s6 ], + [ s7 , s6 , s4 , s6 , s7 ],]) + + + aL = aR[::-1, ::-1] + aB = aT[::-1, ::-1] + + + + + a_components[0, :,:, n, m] = aR[:,:] + a_components[1, :,:, n, m] = aT[:,:] + a_components[2, :,:, n, m] = aL[:,:] + a_components[3, :,:, n, m] = aB[:,:] + + ##============================= + ## Apply bfe to image + ##============================= + + ## pad and expand kernels + ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior + + array_pad = self.saturate(im.copy()).array[4:-4,4:-4] # img of interest 4088x4088 + array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array + + + dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] + + + ### run in sub grids to reduce memory + + ## pad and expand kernels + t = np.zeros((bin_size*n_sub, n_sub)) + for row in range(t.shape[0]): + t[row, row//(bin_size) ] =1 + + + + for gj in range(num_grids): + for gi in range(num_grids): + + a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) + + + for comp in range(4): + for j in range(2*nbfe+1): + for i in range(2*nbfe+1): + tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid + a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') + + #convolve aX_ij with Q_ij + for comp in range(4): + for dy in range(-nbfe, nbfe+1): + for dx in range(-nbfe, nbfe+1): + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ + *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] + + dj = int(np.sin(comp*np.pi/2)) + di = int(np.cos(comp*np.pi/2)) + + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ + array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) + + im.array[:,:] -= dQ_components.sum(axis=0) + im.array[:,1:] += dQ_components[0][:,:-1] + im.array[1:,:] += dQ_components[1][:-1,:] + im.array[:,:-1] += dQ_components[2][:,1:] + im.array[:-1,:] += dQ_components[3][1:,:] + + + return im + + def get_eff_sky_bg(self, pointing=None): + """ + Calculate effective sky background per pixel for nominal roman pixel scale. + + Input + pointing : Pointing object + radec : World coordinate position of image + """ + if pointing is None: + pointing = self.pointing + sky_level = roman.getSkyLevel(pointing.bpass, world_pos=pointing.radec, date=pointing.date) + sky_level *= (1.0 + roman.stray_light_fraction) * roman.pixel_scale**2 + + return sky_level + + def translate_cvz(self, orig_radec, field_ra=9.5, field_dec=-44, cvz_ra=61.24, cvz_dec=-48.42): + + ra = orig_radec.ra/galsim.degrees-field_ra + dec = orig_radec.dec/galsim.degrees-field_dec + ra += cvz_ra / np.cos(cvz_dec*np.pi/180) + dec += cvz_dec + return galsim.CelestialCoord(ra*galsim.degrees,dec*galsim.degrees) + + def setup_sky(self, im, pointing=None, force_cvz=False, stray_light=False, thermal_background=False): + """ + Setup sky + + First we get the amount of zodaical light for a position corresponding to the position of + the object. The results are provided in units of e-/arcsec^2, using the default Roman + exposure time since we did not explicitly specify one. Then we multiply this by a factor + >1 to account for the amount of stray light that is expected. If we do not provide a date + for the observation, then it will assume that it's the vernal equinox (sun at (0,0) in + ecliptic coordinates) in 2025. + + Input + im : Image + pointing : Pointing object + radec : World coordinate position of image + local_wcs : Local WCS + """ + if pointing is None: + pointing = self.pointing + if self.df is None: + self.dark_current_ = roman.dark_current * pointing.exptime + else: + self.dark_current_ = roman.dark_current * pointing.exptime + self.df['DARK'][:, :].flatten() * pointing.exptime + if self.df is None: + self.gain = roman.gain + else: + self.gain = self.df['GAIN'][:,:] + self.read_noise = galsim.GaussianNoise(self.rng, sigma=roman.read_noise) + + # Build current specification sky level if sky level not given + if force_cvz: + radec = self.translate_cvz(pointing.radec) + else: + radec = pointing.radec + + sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) + self.logger.debug('Adding sky_level = %s',sky_level) + if stray_light: + self.logger.debug('Stray light fraction = %s',roman.stray_light_fraction) + sky_level *= (1.0 + roman.stray_light_fraction) + + # Make a image of the sky that takes into account the spatially variable pixel scale. Note + # that makeSkyImage() takes a bit of time. If you do not care about the variable pixel + # scale, you could simply compute an approximate sky level in e-/pix by multiplying + # sky_level by roman.pixel_scale**2, and add that to final_image. + + # Create sky image + self.sky = galsim.Image(bounds=im.bounds, wcs=pointing.WCS) + pointing.WCS.makeSkyImage(self.sky, sky_level) + + # This image is in units of e-/pix. Finally we add the expected thermal backgrounds in this + # band. These are provided in e-/pix/s, so we have to multiply by the exposure time. + if thermal_background: + tb = roman.thermal_backgrounds[pointing.filter] * pointing.exptime + self.logger.debug('Adding thermal background: %s',tb) + self.sky += tb + + # Median of dark current is used here instead of mean since hot pixels contribute significantly to the mean. + # Stastistics of dark current for the current test detector file: (mean, std, median, max) ~ (35, 3050, 0.008, 1.2E6) (e-/p) + # Hot pixels could be removed in further analysis using the dq array. + self.sky_mean = np.mean(np.round((np.round(self.sky.array) + round(np.median(self.dark_current_)))/ np.mean(self.gain))) + + self.sky.addNoise(self.noise) + + def add_background(self, im, save_diff=False, draw_method='phot'): + """ + Add backgrounds to image (sky, thermal). + + First we get the amount of zodaical light for a position corresponding to the position of + the object. The results are provided in units of e-/arcsec^2, using the default Roman + exposure time since we did not explicitly specify one. Then we multiply this by a factor + >1 to account for the amount of stray light that is expected. If we do not provide a date + for the observation, then it will assume that it's the vernal equinox (sun at (0,0) in + ecliptic coordinates) in 2025. + + Input + im : Image + """ + + # If requested, dump an initial fits image to disk for diagnostics + if save_diff: + orig = im.copy() + orig.write('orig.fits') + + if draw_method != 'phot': + im.addNoise(self.noise) + + # Adding sky level to the image. + im += self.sky[self.sky.bounds & im.bounds] + + # If requested, dump a post-change fits image to disk for diagnostics + if save_diff: + prev = im.copy() + diff = prev-orig + diff.write('sky_a.fits') + + return im + + def recip_failure(self, im, exptime=None, alpha=roman.reciprocity_alpha, base_flux=1.0): + """ + Introduce reciprocity failure to image. + + Reciprocity, in the context of photography, is the inverse relationship between the + incident flux (I) of a source object and the exposure time (t) required to produce a given + response(p) in the detector, i.e., p = I*t. However, in NIR detectors, this relation does + not hold always. The pixel response to a high flux is larger than its response to a low + flux. This flux-dependent non-linearity is known as 'reciprocity failure', and the + approximate amount of reciprocity failure for the Roman detectors is known, so we can + include this detector effect in our images. + + Input + im : image + exptime : Exposure time + alpha : Reciprocity alpha + base_flux : Base flux + """ + + if exptime is None: + exptime = self.pointing.exptime + + # Add reciprocity effect + im.addReciprocityFailure(exp_time=exptime, alpha=alpha, base_flux=base_flux) + + return im + + def dark_current(self, im, exptime=None): + """ + Adding dark current to the image. + + Even when the detector is unexposed to any radiation, the electron-hole pairs that + are generated within the depletion region due to finite temperature are swept by the + high electric field at the junction of the photodiode. This small reverse bias + leakage current is referred to as 'dark current'. It is specified by the average + number of electrons reaching the detectors per unit time and has an associated + Poisson noise since it is a random event. + + Input + im : image + """ + + if exptime is None: + exptime = self.pointing.exptime + + if self.df is None: + self.dark_current_ = roman.dark_current * exptime + else: + self.dark_current_ = roman.dark_current * exptime + self.df['DARK'][:, :].flatten() * exptime + + if self.df is None: + self.im_dark = im.copy() + dark_current_ = self.dark_current_ + dark_noise = galsim.DeviateNoise(galsim.PoissonDeviate(self.rng, dark_current_)) + im.addNoise(dark_noise) + self.im_dark = im - self.im_dark + + else: + + dark_current_ = self.dark_current_.clip(0) + + # opt for numpy random geneator instead for speed + self.im_dark = self.rng_np.poisson(dark_current_).reshape(im.array.shape).astype(im.dtype) + im.array[:,:] += self.im_dark + + # NOTE: Sky level and dark current might appear like a constant background that can be + # simply subtracted. However, these contribute to the shot noise and matter for the + # non-linear effects that follow. Hence, these must be included at this stage of the + # image generation process. We subtract these backgrounds in the end. + + return im + + def saturate(self, im, saturation=100000): + """ + Clip the saturation level + Input + im : image + SATURATE[4096,4096] : saturation map + """ + + if self.df is None: + saturation_array = np.ones_like(im.array) * saturation + else: + saturation_array = self.df['SATURATE'][:,:] #4096x4096 array + where_sat = np.where(im.array > saturation_array) + im.array[ where_sat ] = saturation_array[ where_sat ] + + return im + + def deadpix(self, im): + """ + Apply dead pixel mask + Input + im : image + BADPIX[4096,4096] : bit mask with the first bit flags dead pixels + """ + + dead_mask = self.df['BADPIX'][:,:]&1 #4096x4096 array + im.array[ dead_mask>0 ]=0 + + return im + + def vtpe(self, im): + """ + Apply vertical trailing pixel effect. + The vertical trailing pixel effect (VTPE) is a non-linear effect that is + related to readout patterns. + Q'[j,i] = Q[j,i] + f( Q[j,i] - Q[j-1, i] ), + where f( dQ ) = dQ ( a + b * ln(1 + |dQ|/dQ0) ) + Input + im : image + VTPE[0,512,512] : coefficient a binned in 8x8 + VTPE[1,512,512] : coefficient a + VTPE[2,512,512] : coefficient dQ0 + """ + + # expand 512x512 arrays to 4096x4096 + + t = np.zeros((4096, 512)) + for row in range(t.shape[0]): + t[row, row//8] =1 + a_vtpe = t.dot(self.df['VTPE'][0,:,:][0]).dot(t.T) + ## NaN check + if np.isnan(a_vtpe).any(): + print("vtpe skipped due to NaN in file") + return im + b_vtpe = t.dot(self.df['VTPE'][1,:,:][0]).dot(t.T) + dQ0 = t.dot(self.df['VTPE'][2,:,:][0]).dot(t.T) + + dQ = im.array - np.roll(im.array, 1, axis=0) + dQ[0,:] *= 0 + + im.array[:,:] += dQ * ( a_vtpe + b_vtpe * np.log( 1. + np.abs(dQ)/dQ0 )) + return im + + def add_persistence(self, im, pointing=None): + """ + Applying the persistence effect. + + Even after reset, some charges from prior illuminations are trapped in defects of semiconductors. + Trapped charges are gradually released and generate the flux-dependent persistence signal. + Here we adopt the same fermi-linear model to describe the illumination dependence and time dependence + of the persistence effect for all SCAs. + + Input + im : image + pointing : pointing object + """ + if pointing is None: + pointing = self.pointing + + # load the dithers of sky images that were simulated + dither_sca_array=np.loadtxt(self.params['image']['dither_from_file']).astype(int) + + # select adjacent exposures for the same sca (within 10*roman.exptime) + dither_list_selected = dither_sca_array[dither_sca_array[:,1] == pointing.sca, 0] + dither_list_selected = dither_list_selected[ np.abs(dither_list_selected - pointing.visit) < 10 ] + p_list = np.array([get_pointing(self.params, i, self.sca) for i in dither_list_selected]) + dt_list = np.array([(pointing.date - p.date).total_seconds() for p in p_list]) + p_pers = p_list[ np.where((dt_list > 0) & (dt_list < pointing.exptime*10))] + + if self.df is None: + #iterate over previous exposures + for p in p_pers: + dt = (pointing.date - p.date).total_seconds() - pointing.exptime/2 ##avg time since end of exposures + self.params['output']['file_name']['items'] = [p.filter, p.visit, p.sca] + imfilename = ParseValue(self.params['output'], 'file_name', self.params, str)[0] + fn = os.path.join(self.params['output']['dir'], imfilename) + + # [TODO] + if not os.path.exists(fn): + continue + + ## apply all the effects that occured before persistence on the previouse exposures + ## since max of the sky background is of order 100, it is thus negligible for persistence + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4088, ymax=4088) + x = galsim.Image(bound_pad) + x.array[:,:] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] + x = self.recip_failure(x) + + x = x.clip(0) ##remove negative stimulus + + im.array[:,:] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * pointing.exptime + + else: + + #setup parameters for persistence + Q01 = self.df['PERSIST'].read_header()['Q01'] + Q02 = self.df['PERSIST'].read_header()['Q02'] + Q03 = self.df['PERSIST'].read_header()['Q03'] + Q04 = self.df['PERSIST'].read_header()['Q04'] + Q05 = self.df['PERSIST'].read_header()['Q05'] + Q06 = self.df['PERSIST'].read_header()['Q06'] + alpha = self.df['PERSIST'].read_header()['ALPHA'] + + #iterate over previous exposures + for p in p_pers: + dt = (pointing.date - p.date).total_seconds() - pointing.exptime/2 ##avg time since end of exposures + fac_dt = (pointing.exptime/2.) / dt ##linear time dependence (approximate until we get t1 and Delat t of the data) + self.params['output']['file_name']['items'] = [p.filter, p.visit, p.sca] + imfilename = ParseValue(self.params['output'], 'file_name', self.params, str)[0] + fn = os.path.join(self.params['output']['dir'], imfilename) + + # [TODO] + if not os.path.exists(fn): + continue + + ## apply all the effects that occured before persistence on the previouse exposures + ## since max of the sky background is of order 100, it is thus negligible for persistence + ## same for brighter fatter effect + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + x = galsim.Image(bound_pad) + x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] + x = self.qe(x).array[:,:] + + x = x.clip(0.1) ##remove negative and zero stimulus + + ## Do linear interpolation + a = np.zeros(x.shape) + a += ((x < Q01)) * x/Q01 + a += ((x >= Q01) & (x < Q02)) * (Q02-x)/(Q02-Q01) + im.array[:,:] += a * self.df['PERSIST'][0,:,:][0] * fac_dt + + + a = np.zeros(x.shape) + a += ((x >= Q01) & (x < Q02)) * (x-Q01)/(Q02-Q01) + a += ((x >= Q02) & (x < Q03)) * (Q03-x)/(Q03-Q02) + im.array[:,:] += a * self.df['PERSIST'][1,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q02) & (x < Q03)) * (x-Q02)/(Q03-Q02) + a += ((x >= Q03) & (x < Q04)) * (Q04-x)/(Q04-Q03) + im.array[:,:] += a * self.df['PERSIST'][2,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q03) & (x < Q04)) * (x-Q03)/(Q04-Q03) + a += ((x >= Q04) & (x < Q05)) * (Q05-x)/(Q05-Q04) + im.array[:,:] += a * self.df['PERSIST'][3,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q04) & (x < Q05)) * (x-Q04)/(Q05-Q04) + a += ((x >= Q05) & (x < Q06)) * (Q06-x)/(Q06-Q05) + im.array[:,:] += a * self.df['PERSIST'][4,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q05) & (x < Q06)) * (x-Q05)/(Q06-Q05) + a += ((x >= Q06)) * (x/Q06)**alpha ##avoid fractional power of negative values + im.array[:,:] += a * self.df['PERSIST'][5,:,:][0] * fac_dt + + + return im + + def nonlinearity(self, im, NLfunc=roman.NLfunc): + """ + Applying a quadratic non-linearity. + + Note that users who wish to apply some other nonlinearity function (perhaps for other NIR + detectors, or for CCDs) can use the more general nonlinearity routine, which uses the + following syntax: + final_image.applyNonlinearity(NLfunc=NLfunc) + with NLfunc being a callable function that specifies how the output image pixel values + should relate to the input ones. + + Input + im : Image + NLfunc : Nonlinearity function + """ + + # Apply the Roman nonlinearity routine, which knows all about the nonlinearity expected in + # the Roman detectors. Alternately, use a user-provided function. + if self.df is None: + im.applyNonlinearity(NLfunc=NLfunc) + else: + im.array[:,:] -= self.df['CNL'][0,:,:][0] * im.array**2 +\ + self.df['CNL'][1,:,:][0] * im.array**3 +\ + self.df['CNL'][2,:,:][0] * im.array**4 + + return im + + def interpix_cap(self, im, kernel=roman.ipc_kernel): + """ + Including Interpixel capacitance + + The voltage read at a given pixel location is influenced by the charges present in the + neighboring pixel locations due to capacitive coupling of sense nodes. This interpixel + capacitance effect is modeled as a linear effect that is described as a convolution of a + 3x3 kernel with the image. The Roman IPC routine knows about the kernel already, so the + user does not have to supply it. + + Input + im : image + kernel : Interpixel capacitance kernel + """ + + # Apply interpixel capacitance + if self.df is None: + im.applyIPC(kernel, edge_treatment='extend', fill_value=None) + else: + # pad the array by one pixel at the four edges + num_grids = 4 ### num_grids <= 8 + grid_size = 4096//num_grids + + array_pad = im.array[4:-4,4:-4] #it's an array instead of img + array_pad = np.pad(array_pad, [(5,5),(5,5)], mode='symmetric') #4098x4098 array + + K = self.df['IPC'][:, :, :, :] ##3,3,512, 512 + + t = np.zeros((grid_size, 512)) + for row in range(t.shape[0]): + t[row, row//( grid_size//512) ] =1 + + array_out = np.zeros( (4096, 4096)) + ##split job in sub_grids to reduce memory + for gj in range(num_grids): + for gi in range(num_grids): + K_pad = np.zeros( (3,3, grid_size+2, grid_size+2) ) + + for j in range(3): + for i in range(3): + tmp = (t.dot(K[j,i,:,:])).dot(t.T) #grid_sizexgrid_size + K_pad[j,i,:,:] = np.pad(tmp, [(1,1),(1,1)], mode='symmetric') + + for dy in range(-1, 2): + for dx in range(-1,2): + + array_out[ gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ + +=K_pad[ 1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size ] \ + *array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] + + im.array[:,:] = array_out + + return im + + + def add_read_noise(self, im): + """ + Adding read noise + + Read noise is the noise due to the on-chip amplifier that converts the charge into an + analog voltage. We already applied the Poisson noise due to the sky level, so read noise + should just be added as Gaussian noise + + Input + im : image + """ + + # Create noise realisation and apply it to image + if self.df is None: + self.im_read = im.copy() + im.addNoise(self.read_noise) + self.im_read = im - self.im_read + # self.sky.addNoise(self.read_noise) + else: + # use numpy random generator to draw 2-d noise map + read_noise = self.df['READ'][2,:,:].flatten() #flattened 4096x4096 array + self.im_read = self.rng_np.normal(loc=0., scale=read_noise).reshape(im.array.shape).astype(im.dtype) + im.array[:,:] += self.im_read + + # noise_array = self.rng_np.normal(loc=0., scale=read_noise) + # 4088x4088 img + # self.sky.array[:,:] += noise_array.reshape(im.array.shape)[4:-4, 4:-4].astype(self.sky.dtype) + + return im + + def e_to_ADU(self, im): + """ + We divide by the gain to convert from e- to ADU. Currently, the gain value in the Roman + module is just set to 1, since we don't know what the exact gain will be, although it is + expected to be approximately 1. Eventually, this may change when the camera is assembled, + and there may be a different value for each SCA. For now, there is just a single number, + which is equal to 1. + + Input + im : image + """ + if self.df is None: + return im / roman.gain + else: + bias = self.df['BIAS'][:,:] #4096x4096 img + t = np.zeros((4096, 32)) + for row in range(t.shape[0]): + t[row, row//128] =1 + gain_expand = (t.dot(self.gain)).dot(t.T) #4096x4096 gain img + im.array[:,:] = im.array/gain_expand + bias + return im + + def add_gain(self, im): + """ + We divide by the gain to convert from e- to ADU. + Input + im : image + GAIN : 32x32 float img in unit of e-/adu, mean(GAIN)~ 1.6 + """ + + gain = self.df['GAIN'][:, :] #32x32 img + + t = np.zeros((4096, 32)) + for row in range(t.shape[0]): + t[row, row//128] =1 + gain_expand = (t.dot(gain)).dot(t.T) #4096x4096 gain img + im.array[:,:] /= gain_expand + return im + + def add_bias(self, im): + """ + Add the voltage bias. + Input + im : image + BIAS : 4096x4096 uint16 bias img (in unit of DN), mean(bias) ~ 6.7k + """ + + bias = self.df['BIAS'][:,:] #4096x4096 img + + im.array[:,:] += bias + return im + + def finalize_sky_im(self,im, pointing=None): + """ + Finalize sky background for subtraction from final image. Add dark current, + convert to analog voltage, and quantize. + + Input + im : sky image + """ + + if pointing is None: + pointing = self.pointing + + if self.df is None: + im.quantize() + im += self.im_dark + im = self.saturate(im) + im += self.im_read + im = self.e_to_ADU(im) + im.quantize() + else: + + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + im_pad = galsim.Image(bound_pad) + im_pad.array[4:-4, 4:-4] = im.array[:,:] + + im_pad = self.qe(im_pad) + im_pad = self.bfe(im_pad) + im_pad = self.add_persistence(im_pad, pointing) + im_pad.quantize() + im_pad += self.im_dark + im_pad = self.saturate(im_pad) + im_pad = self.nonlinearity(im_pad) + im_pad = self.interpix_cap(im_pad) + im_pad = self.deadpix(im_pad) + im_pad = self.vtpe(im_pad) + im_pad += self.im_read + im_pad = self.add_gain(im_pad) + im_pad = self.add_bias(im_pad) + im_pad.quantize() + # output 4088x4088 img in uint16 + im.array[:,:] = im_pad.array[4:-4, 4:-4] + im = galsim.Image(im, dtype=np.uint16) + + return im \ No newline at end of file diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 7764770d..171935f8 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -6,7 +6,10 @@ from galsim.config import RegisterImageType from galsim.config.image_scattered import ScatteredImageBuilder from galsim.image import Image +from astropy.time import Time +import numpy as np +from .detector_effects import detector_effects class RomanSCAImageBuilder(ScatteredImageBuilder): @@ -51,16 +54,20 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): ] req = {"SCA": int, "filter": str, "mjd": float, "exptime": float} opt = { - "draw_method": str, - "stray_light": bool, - "thermal_background": bool, - "reciprocity_failure": bool, - "dark_current": bool, - "nonlinearity": bool, - "ipc": bool, - "read_noise": bool, - "sky_subtract": bool, - "ignore_noise": bool, + 'draw_method' : str, + 'stray_light' : bool, + 'thermal_background' : bool, + 'reciprocity_failure' : bool, + 'dark_current' : bool, + 'nonlinearity' : bool, + 'ipc' : bool, + 'read_noise' : bool, + 'sky_subtract' : bool, + 'ignore_noise' : bool, + 'sca_filepath' : str, + 'dither_from_file': str, + 'sca_filepath': str, + 'save_diff': bool } params = galsim.config.GetAllParams(config, base, req=req, opt=opt, ignore=ignore + extra_ignore)[0] @@ -95,6 +102,19 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): # # GalSim expects a wcs in the image field. # config['wcs'] = wcs + self.rng = galsim.config.GetRNG(config, base) + self.visit = int(base['input']['obseq_data']['visit']) + + self.sca_filepath = params.get('sca_filepath', None) + self.effects = detector_effects(params = base, + visit = self.visit, + sca = self.sca, + filter = self.filter, + logger = logger, + rng = self.rng, + rng_iter = self.visit * self.sca, + sca_filepath = self.sca_filepath) + # If user hasn't overridden the bandpass to use, get the standard one. if "bandpass" not in config: base["bandpass"] = galsim.config.BuildBandpass(base["image"], "bandpass", base, logger=logger) @@ -193,7 +213,10 @@ def buildImage(self, config, base, image_num, obj_num, logger): ) logger.debug("image %d: Overlap = %s", image_num, str(bounds)) full_image[bounds] += stamps[k][bounds] - stamps = None + stamps=None + + # # [TODO] + # break # # Bring the image so far up to a flat noise variance # current_var = FlattenNoiseVariance( @@ -201,6 +224,111 @@ def buildImage(self, config, base, image_num, obj_num, logger): return full_image, None + # def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): + # """Add the final noise to a Scattered image + + # Parameters: + # image: The image onto which to add the noise. + # config: The configuration dict for the image field. + # base: The base configuration dict. + # image_num: The current image number. + # obj_num: The first object number in the image. + # current_var: The current noise variance in each postage stamps. + # logger: If given, a logger object to log progress. + # """ + # # check ignore noise + # if self.ignore_noise: + # return + + # base['current_noise_image'] = base['current_image'] + # wcs = base['wcs'] + # bp = base['bandpass'] + # # rng = galsim.config.GetRNG(config, base) + # logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) + + # # Things that will eventually be subtracted (if sky_subtract) will have their expectation + # # value added to sky_image. So technically, this includes things that aren't just sky. + # # E.g. includes dark_current and thermal backgrounds. + # sky_image = image.copy() + # sky_level = roman.getSkyLevel(bp, world_pos=wcs.toWorld(image.true_center)) + # logger.debug('Adding sky_level = %s',sky_level) + # if self.stray_light: + # logger.debug('Stray light fraction = %s',roman.stray_light_fraction) + # sky_level *= (1.0 + roman.stray_light_fraction) + # wcs.makeSkyImage(sky_image, sky_level) + + # # The other background is the expected thermal backgrounds in this band. + # # These are provided in e-/pix/s, so we have to multiply by the exposure time. + # if self.thermal_background: + # tb = roman.thermal_backgrounds[self.filter] * self.exptime + # logger.debug('Adding thermal background: %s',tb) + # sky_image += roman.thermal_backgrounds[self.filter] * self.exptime + + # # The image up to here is an expectation value. + # # Realize it as an integer number of photons. + # poisson_noise = galsim.noise.PoissonNoise(self.rng) + # if self.draw_method == 'phot': + # logger.debug("Adding poisson noise to sky photons") + # sky_image1 = sky_image.copy() + # sky_image1.addNoise(poisson_noise) + # image.quantize() # In case any profiles used InterpolatedImage, in which case + # # the image won't necessarily be integers. + # image += sky_image1 + # else: + # logger.debug("Adding poisson noise") + # image += sky_image + # image.addNoise(poisson_noise) + + # # Apply the detector effects here. Not all of these are "noise" per se, but they + # # happen interspersed with various noise effects, so apply them all in this step. + + # # Note: according to Gregory Mosby & Bernard J. Rauscher, the following effects all + # # happen "simultaneously" in the photo diodes: dark current, persistence, + # # reciprocity failure (aka CRNL), burn in, and nonlinearity (aka CNL). + # # Right now, we just do them in some order, but this could potentially be improved. + # # The order we chose is historical, matching previous recommendations, but Mosby and + # # Rauscher don't seem to think those recommendations are well-motivated. + + # # TODO: Add burn-in and persistence here. + + # if self.reciprocity_failure: + # logger.debug("Applying reciprocity failure") + # roman.addReciprocityFailure(image) + + # if self.dark_current: + # dc = roman.dark_current * self.exptime + # logger.debug("Adding dark current: %s",dc) + # sky_image += dc + # dark_noise = galsim.noise.DeviateNoise(galsim.random.PoissonDeviate(self.rng, dc)) + # image.addNoise(dark_noise) + + # if self.nonlinearity: + # logger.debug("Applying classical nonlinearity") + # roman.applyNonlinearity(image) + + # # Mosby and Rauscher say there are two read noises. One happens before IPC, the other + # # one after. + # # TODO: Add read_noise1 + # if self.ipc: + # logger.debug("Applying IPC") + # roman.applyIPC(image) + + # if self.read_noise: + # logger.debug("Adding read noise %s", roman.read_noise) + # image.addNoise(galsim.GaussianNoise(self.rng, sigma=roman.read_noise)) + + # logger.debug("Applying gain %s",roman.gain) + # image /= roman.gain + + # # Make integer ADU now. + # image.quantize() + + # if self.sky_subtract: + # logger.debug("Subtracting sky image") + # sky_image /= roman.gain + # sky_image.quantize() + # image -= sky_image + def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): """Add the final noise to a Scattered image @@ -217,44 +345,129 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) if self.ignore_noise: return - base["current_noise_image"] = base["current_image"] - wcs = base["wcs"] - bp = base["bandpass"] - rng = galsim.config.GetRNG(config, base) - logger.info("image %d: Start RomanSCA detector effects", base.get("image_num", 0)) + base['current_noise_image'] = base['current_image'] + wcs = base['wcs'] + bp = base['bandpass'] + # rng = galsim.config.GetRNG(config, base) + logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) # Things that will eventually be subtracted (if sky_subtract) will have their expectation # value added to sky_image. So technically, this includes things that aren't just sky. # E.g. includes dark_current and thermal backgrounds. - sky_image = image.copy() - sky_level = roman.getSkyLevel(bp, world_pos=wcs.toWorld(image.true_center)) - logger.debug("Adding sky_level = %s", sky_level) - if self.stray_light: - logger.debug("Stray light fraction = %s", roman.stray_light_fraction) - sky_level *= 1.0 + roman.stray_light_fraction - wcs.makeSkyImage(sky_image, sky_level) + # sky_image = image.copy() + # sky_level = roman.getSkyLevel(bp, world_pos=wcs.toWorld(image.true_center)) + # logger.debug('Adding sky_level = %s',sky_level) + # if self.stray_light: + # logger.debug('Stray light fraction = %s',roman.stray_light_fraction) + # sky_level *= (1.0 + roman.stray_light_fraction) + # wcs.makeSkyImage(sky_image, sky_level) # The other background is the expected thermal backgrounds in this band. # These are provided in e-/pix/s, so we have to multiply by the exposure time. - if self.thermal_background: - tb = roman.thermal_backgrounds[self.filter] * self.exptime - logger.debug("Adding thermal background: %s", tb) - sky_image += roman.thermal_backgrounds[self.filter] * self.exptime + # if self.thermal_background: + # tb = roman.thermal_backgrounds[self.filter] * self.exptime + # logger.debug('Adding thermal background: %s',tb) + # sky_image += roman.thermal_backgrounds[self.filter] * self.exptime + + self.effects.setup_sky(image, force_cvz=self.effects.force_cvz, stray_light=self.stray_light, thermal_background=self.thermal_background) + # [TODO] quantize() at this step? # The image up to here is an expectation value. # Realize it as an integer number of photons. - poisson_noise = galsim.noise.PoissonNoise(rng) - if self.draw_method == "phot": - logger.debug("Adding poisson noise to sky photons") - sky_image1 = sky_image.copy() - sky_image1.addNoise(poisson_noise) - image.quantize() # In case any profiles used InterpolatedImage, in which case - # the image won't necessarily be integers. - image += sky_image1 + # poisson_noise = galsim.noise.PoissonNoise(self.rng) + # if self.draw_method == 'phot': + # logger.debug("Adding poisson noise to sky photons") + # sky_image1 = sky_image.copy() + # sky_image1.addNoise(poisson_noise) + # image.quantize() # In case any profiles used InterpolatedImage, in which case + # # the image won't necessarily be integers. + # image += sky_image1 + # else: + # logger.debug("Adding poisson noise") + # image += sky_image + # image.addNoise(poisson_noise) + image = self.effects.add_background(image, draw_method=self.draw_method) + + if self.sca_filepath is not None: + ## create padded image + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + im_pad = galsim.Image(bound_pad) + im_pad.array[4:-4, 4:-4] = image.array[:,:] + self.effects.set_diff(im_pad) + im_pad = self.effects.qe(im_pad) + self.effects.diff('qe', im_pad) + + im_pad = self.effects.bfe(im_pad) + self.effects.diff('bfe', im_pad) + + im_pad = self.effects.add_persistence(im_pad) + self.effects.diff('pers', im_pad) + + im_pad.quantize() + self.effects.diff('quantize1', im_pad) + + im_pad = self.effects.dark_current(im_pad) + self.effects.diff('dark', im_pad) + + im_pad = self.effects.saturate(im_pad) + self.effects.diff('sat', im_pad) + + im_pad = self.effects.nonlinearity(im_pad) + self.effects.diff('cnl', im_pad) + + im_pad = self.effects.interpix_cap(im_pad) + self.effects.diff('ipc', im_pad) + + im_pad = self.effects.deadpix(im_pad) + self.effects.diff('deadpix', im_pad) + + im_pad = self.effects.vtpe(im_pad) + self.effects.diff('vtpe', im_pad) + + im_pad = self.effects.add_read_noise(im_pad) + self.effects.diff('read', im_pad) + + im_pad = self.effects.add_gain(im_pad) + self.effects.diff('gain', im_pad) + + im_pad = self.effects.add_bias(im_pad) + self.effects.diff('bias', im_pad) + + im_pad.quantize() + self.effects.diff('quantize2', im_pad) + + # output 4088x4088 img in uint16 + image.array[:,:] = im_pad.array[4:-4, 4:-4] + + # [TODO] + # # data quality image + # # 0x1 -> non-responsive + # # 0x2 -> hot pixel + # # 0x4 -> very hot pixel + # # 0x8 -> adjacent to pixel with strange response + # # 0x10 -> low CDS, high total noise pixel (may have strange settling behaviors, not recommended for precision applications) + # # 0x20 -> CNL fit went down to the minimum number of points (remaining degrees of freedom = 0) + # # 0x40 -> no solid-waffle solution for this region (set gain value to array median). normally occurs in a few small regions of some SCAs with lots of bad pixels. [recommend not to use these regions for WL analysis] + # # 0x80 -> wt==0 + # dq = self.df['BADPIX'][4:4092, 4:4092] + # # get weight map + # if wt is not None: + # dq[wt==0] += 128 + + # sky_noise = self.sky.copy() + # sky_noise = self.finalize_sky_im(sky_noise, pointing) + else: - logger.debug("Adding poisson noise") - image += sky_image - image.addNoise(poisson_noise) + image = self.effects.recip_failure(image) # Introduce reciprocity failure to image + image.quantize() # At this point in the image generation process, an integer number of photons gets detected + image = self.effects.dark_current(image) # Add dark current to image + image = self.effects.add_persistence(image) + image = self.effects.saturate(image) + image= self.effects.nonlinearity(image) # Apply nonlinearity + image = self.effects.interpix_cap(image) # Introduce interpixel capacitance to image. + image = self.effects.add_read_noise(image) + image = self.effects.e_to_ADU(image) # Convert electrons to ADU # Apply the detector effects here. Not all of these are "noise" per se, but they # happen interspersed with various noise effects, so apply them all in this step. @@ -268,42 +481,47 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) # TODO: Add burn-in and persistence here. - if self.reciprocity_failure: - logger.debug("Applying reciprocity failure") - roman.addReciprocityFailure(image) + # if self.reciprocity_failure: + # logger.debug("Applying reciprocity failure") + # roman.addReciprocityFailure(image) - if self.dark_current: - dc = roman.dark_current * self.exptime - logger.debug("Adding dark current: %s", dc) - sky_image += dc - dark_noise = galsim.noise.DeviateNoise(galsim.random.PoissonDeviate(rng, dc)) - image.addNoise(dark_noise) + # if self.dark_current: + # dc = roman.dark_current * self.exptime + # logger.debug("Adding dark current: %s",dc) + # sky_image += dc + # dark_noise = galsim.noise.DeviateNoise(galsim.random.PoissonDeviate(self.rng, dc)) + # image.addNoise(dark_noise) - if self.nonlinearity: - logger.debug("Applying classical nonlinearity") - roman.applyNonlinearity(image) + # if self.nonlinearity: + # logger.debug("Applying classical nonlinearity") + # roman.applyNonlinearity(image) - # Mosby and Rauscher say there are two read noises. One happens before IPC, the other - # one after. - # TODO: Add read_noise1 - if self.ipc: - logger.debug("Applying IPC") - roman.applyIPC(image) + # # Mosby and Rauscher say there are two read noises. One happens before IPC, the other + # # one after. + # # TODO: Add read_noise1 + # if self.ipc: + # logger.debug("Applying IPC") + # roman.applyIPC(image) - if self.read_noise: - logger.debug("Adding read noise %s", roman.read_noise) - image.addNoise(galsim.GaussianNoise(rng, sigma=roman.read_noise)) + # if self.read_noise: + # logger.debug("Adding read noise %s", roman.read_noise) + # image.addNoise(galsim.GaussianNoise(self.rng, sigma=roman.read_noise)) - logger.debug("Applying gain %s", roman.gain) - image /= roman.gain + # logger.debug("Applying gain %s",roman.gain) + # image /= roman.gain # Make integer ADU now. image.quantize() + # if self.sky_subtract: + # logger.debug("Subtracting sky image") + # sky_image /= roman.gain + # sky_image.quantize() + # image -= sky_image + if self.sky_subtract: logger.debug("Subtracting sky image") - sky_image /= roman.gain - sky_image.quantize() + sky_image = self.effects.finalize_sky_im(self.effects.sky.copy()) image -= sky_image From 48b3560efca5e7bca59e01933c925317d799f526 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Fri, 18 Oct 2024 16:50:08 -0400 Subject: [PATCH 02/26] fix clip calls in add_persistence --- roman_imsim/detector_effects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/roman_imsim/detector_effects.py b/roman_imsim/detector_effects.py index ba116e17..60e40024 100644 --- a/roman_imsim/detector_effects.py +++ b/roman_imsim/detector_effects.py @@ -551,7 +551,8 @@ def add_persistence(self, im, pointing=None): x.array[:,:] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] x = self.recip_failure(x) - x = x.clip(0) ##remove negative stimulus + # x = x.clip(0) ##remove negative stimulus + x.array.clip(0) ##remove negative stimulus im.array[:,:] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * pointing.exptime From 4540ad029d20abe3ac86e23b8080b9e249747597 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 28 May 2025 16:51:37 -0400 Subject: [PATCH 03/26] remove some commented codes in sca.py --- roman_imsim/sca.py | 171 --------------------------------------------- 1 file changed, 171 deletions(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 171935f8..f9925bbc 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -215,119 +215,12 @@ def buildImage(self, config, base, image_num, obj_num, logger): full_image[bounds] += stamps[k][bounds] stamps=None - # # [TODO] - # break - # # Bring the image so far up to a flat noise variance # current_var = FlattenNoiseVariance( # base, full_image, stamps, current_vars, logger) return full_image, None - # def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): - # """Add the final noise to a Scattered image - - # Parameters: - # image: The image onto which to add the noise. - # config: The configuration dict for the image field. - # base: The base configuration dict. - # image_num: The current image number. - # obj_num: The first object number in the image. - # current_var: The current noise variance in each postage stamps. - # logger: If given, a logger object to log progress. - # """ - # # check ignore noise - # if self.ignore_noise: - # return - - # base['current_noise_image'] = base['current_image'] - # wcs = base['wcs'] - # bp = base['bandpass'] - # # rng = galsim.config.GetRNG(config, base) - # logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) - - # # Things that will eventually be subtracted (if sky_subtract) will have their expectation - # # value added to sky_image. So technically, this includes things that aren't just sky. - # # E.g. includes dark_current and thermal backgrounds. - # sky_image = image.copy() - # sky_level = roman.getSkyLevel(bp, world_pos=wcs.toWorld(image.true_center)) - # logger.debug('Adding sky_level = %s',sky_level) - # if self.stray_light: - # logger.debug('Stray light fraction = %s',roman.stray_light_fraction) - # sky_level *= (1.0 + roman.stray_light_fraction) - # wcs.makeSkyImage(sky_image, sky_level) - - # # The other background is the expected thermal backgrounds in this band. - # # These are provided in e-/pix/s, so we have to multiply by the exposure time. - # if self.thermal_background: - # tb = roman.thermal_backgrounds[self.filter] * self.exptime - # logger.debug('Adding thermal background: %s',tb) - # sky_image += roman.thermal_backgrounds[self.filter] * self.exptime - - # # The image up to here is an expectation value. - # # Realize it as an integer number of photons. - # poisson_noise = galsim.noise.PoissonNoise(self.rng) - # if self.draw_method == 'phot': - # logger.debug("Adding poisson noise to sky photons") - # sky_image1 = sky_image.copy() - # sky_image1.addNoise(poisson_noise) - # image.quantize() # In case any profiles used InterpolatedImage, in which case - # # the image won't necessarily be integers. - # image += sky_image1 - # else: - # logger.debug("Adding poisson noise") - # image += sky_image - # image.addNoise(poisson_noise) - - # # Apply the detector effects here. Not all of these are "noise" per se, but they - # # happen interspersed with various noise effects, so apply them all in this step. - - # # Note: according to Gregory Mosby & Bernard J. Rauscher, the following effects all - # # happen "simultaneously" in the photo diodes: dark current, persistence, - # # reciprocity failure (aka CRNL), burn in, and nonlinearity (aka CNL). - # # Right now, we just do them in some order, but this could potentially be improved. - # # The order we chose is historical, matching previous recommendations, but Mosby and - # # Rauscher don't seem to think those recommendations are well-motivated. - - # # TODO: Add burn-in and persistence here. - - # if self.reciprocity_failure: - # logger.debug("Applying reciprocity failure") - # roman.addReciprocityFailure(image) - - # if self.dark_current: - # dc = roman.dark_current * self.exptime - # logger.debug("Adding dark current: %s",dc) - # sky_image += dc - # dark_noise = galsim.noise.DeviateNoise(galsim.random.PoissonDeviate(self.rng, dc)) - # image.addNoise(dark_noise) - - # if self.nonlinearity: - # logger.debug("Applying classical nonlinearity") - # roman.applyNonlinearity(image) - - # # Mosby and Rauscher say there are two read noises. One happens before IPC, the other - # # one after. - # # TODO: Add read_noise1 - # if self.ipc: - # logger.debug("Applying IPC") - # roman.applyIPC(image) - - # if self.read_noise: - # logger.debug("Adding read noise %s", roman.read_noise) - # image.addNoise(galsim.GaussianNoise(self.rng, sigma=roman.read_noise)) - - # logger.debug("Applying gain %s",roman.gain) - # image /= roman.gain - - # # Make integer ADU now. - # image.quantize() - - # if self.sky_subtract: - # logger.debug("Subtracting sky image") - # sky_image /= roman.gain - # sky_image.quantize() - # image -= sky_image def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): """Add the final noise to a Scattered image @@ -351,23 +244,6 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) # rng = galsim.config.GetRNG(config, base) logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) - # Things that will eventually be subtracted (if sky_subtract) will have their expectation - # value added to sky_image. So technically, this includes things that aren't just sky. - # E.g. includes dark_current and thermal backgrounds. - # sky_image = image.copy() - # sky_level = roman.getSkyLevel(bp, world_pos=wcs.toWorld(image.true_center)) - # logger.debug('Adding sky_level = %s',sky_level) - # if self.stray_light: - # logger.debug('Stray light fraction = %s',roman.stray_light_fraction) - # sky_level *= (1.0 + roman.stray_light_fraction) - # wcs.makeSkyImage(sky_image, sky_level) - - # The other background is the expected thermal backgrounds in this band. - # These are provided in e-/pix/s, so we have to multiply by the exposure time. - # if self.thermal_background: - # tb = roman.thermal_backgrounds[self.filter] * self.exptime - # logger.debug('Adding thermal background: %s',tb) - # sky_image += roman.thermal_backgrounds[self.filter] * self.exptime self.effects.setup_sky(image, force_cvz=self.effects.force_cvz, stray_light=self.stray_light, thermal_background=self.thermal_background) # [TODO] quantize() at this step? @@ -469,56 +345,9 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) image = self.effects.add_read_noise(image) image = self.effects.e_to_ADU(image) # Convert electrons to ADU - # Apply the detector effects here. Not all of these are "noise" per se, but they - # happen interspersed with various noise effects, so apply them all in this step. - - # Note: according to Gregory Mosby & Bernard J. Rauscher, the following effects all - # happen "simultaneously" in the photo diodes: dark current, persistence, - # reciprocity failure (aka CRNL), burn in, and nonlinearity (aka CNL). - # Right now, we just do them in some order, but this could potentially be improved. - # The order we chose is historical, matching previous recommendations, but Mosby and - # Rauscher don't seem to think those recommendations are well-motivated. - - # TODO: Add burn-in and persistence here. - - # if self.reciprocity_failure: - # logger.debug("Applying reciprocity failure") - # roman.addReciprocityFailure(image) - - # if self.dark_current: - # dc = roman.dark_current * self.exptime - # logger.debug("Adding dark current: %s",dc) - # sky_image += dc - # dark_noise = galsim.noise.DeviateNoise(galsim.random.PoissonDeviate(self.rng, dc)) - # image.addNoise(dark_noise) - - # if self.nonlinearity: - # logger.debug("Applying classical nonlinearity") - # roman.applyNonlinearity(image) - - # # Mosby and Rauscher say there are two read noises. One happens before IPC, the other - # # one after. - # # TODO: Add read_noise1 - # if self.ipc: - # logger.debug("Applying IPC") - # roman.applyIPC(image) - - # if self.read_noise: - # logger.debug("Adding read noise %s", roman.read_noise) - # image.addNoise(galsim.GaussianNoise(self.rng, sigma=roman.read_noise)) - - # logger.debug("Applying gain %s",roman.gain) - # image /= roman.gain - # Make integer ADU now. image.quantize() - # if self.sky_subtract: - # logger.debug("Subtracting sky image") - # sky_image /= roman.gain - # sky_image.quantize() - # image -= sky_image - if self.sky_subtract: logger.debug("Subtracting sky image") sky_image = self.effects.finalize_sky_im(self.effects.sky.copy()) From 9aef8d2cfeb01a35523c806178e49bebf59e0e7a Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Sat, 28 Jun 2025 22:08:09 -0400 Subject: [PATCH 04/26] Start refactoring the detector_effects class: * Enable selection of different models for each effect via the configuration YAML. * Implementing a nested class structure in detector_effects.py. * Testing the implementation of QE and BFE effects. --- config/was.yaml | 6 +- roman_imsim/detector_effects.py | 501 ++++++++++++++++++++++---------- roman_imsim/sca.py | 64 ++-- 3 files changed, 398 insertions(+), 173 deletions(-) diff --git a/config/was.yaml b/config/was.yaml index 28e31436..7f3ce2e7 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -89,6 +89,8 @@ image: # sky_subtract: False ignore_noise: False + quantum_efficiency: lab_model + brighter_fatter: lab_model stray_light: True thermal_background: True reciprocity_failure: True @@ -96,9 +98,9 @@ image: nonlinearity: True ipc: True read_noise: True - sky_subtract: True + sky_subtract: False - dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt + # dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt sca_filepath: /hpc/group/cosmology/phy-lsst/roman_sensors save_diff: False diff --git a/roman_imsim/detector_effects.py b/roman_imsim/detector_effects.py index 60e40024..9bc12807 100644 --- a/roman_imsim/detector_effects.py +++ b/roman_imsim/detector_effects.py @@ -81,6 +81,198 @@ def __init__(self, params, visit, sca, filter, logger, rng, rng_iter, sca_filepa self.save_diff = False if 'save_diff' in self.params['image']: self.save_diff = bool(self.params['image']['save_diff']) + + class quantum_efficiency(object): + def __init__(self, params, logger, model='simple_model', sca_filepath=None): + self.model = getattr(self, model) + self.sca_filepath = sca_filepath + self.params = params + self.logger = logger + + def simple_model(self, image): + self.logger.info("Applying the simple QE model...") + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No QE data file provided; a default value of QE = 1 will be used.") + return image + + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.params['SCA']])) + self.logger.info("Applying the QE model derived from lab data…") + image.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array + return image + + def apply(self, image): + image = self.model(image) + return image + + class bfe(object): + def __init__(self, params, logger, model='simple_model', sca_filepath=None, saturation_level=100000): + self.model = getattr(self, model) + self.sca_filepath = sca_filepath + self.params = params + self.logger = logger + self.saturation_level = saturation_level + + def simple_model(sefl, image): + self.logger.info("No bfe effect will be applied.") + return image + + def lab_model(self, image): + """ + Apply brighter-fatter effect. + Brighter fatter effect is a non-linear effect that deflects photons due to the + the eletric field built by the accumulated charges. This effect exists in both + CCD and CMOS detectors and typically percent level change in charge. + The built-in electric field by the charges in pixels tends to repulse charges + to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. + This effect can also be understood effectly as change in pixel area and pixel + boundaries. + BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change + in the detector effect charaterization file. Kernel of the total pixel area, however, + is not sufficient. Image simulation of the brighter fatter effect requires the shift + of the four pixel boundaries. Before we get better data, we solve for the boundary + shift components from the kernel of total pixel area by assumming several symmetric constraints. + Input + im : Image + BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 + """ + if self.sca_filepath is None: + self.logger.warning("No BFE kernel data file provided; no bfe effect will be applied.") + return image + + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.params['SCA']])) + + nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) + bin_size = 128 + n_max = 32 + m_max = 32 + num_grids = 4 + n_sub = n_max//num_grids + m_sub = m_max//num_grids + + ##======================================================================= + ## solve boundary shfit kernel aX components + ##======================================================================= + a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 + a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 + + ##solve aR aT aL aB for each a + for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) + for m in range(m_max): + a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) + + ## assume two parity symmetries + a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. + + r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection + B = (a[2,2], a[3,2], a[2,3], a[3,3], + a[4,2], a[2,4], a[3,4], a[4,4] ) + + A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], + [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], + [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], + [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], + [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], + [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], + [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], + [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) + + + s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] + + aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s3 , -s1 , s1 , s3 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) + + + aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], + [ -s7 , -s6 , -s4 , -s6 , -s7 ], + [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], + [ r*s6 , s5 , s2 , s5 , r*s6 ], + [ s7 , s6 , s4 , s6 , s7 ],]) + + + aL = aR[::-1, ::-1] + aB = aT[::-1, ::-1] + + + + + a_components[0, :,:, n, m] = aR[:,:] + a_components[1, :,:, n, m] = aT[:,:] + a_components[2, :,:, n, m] = aL[:,:] + a_components[3, :,:, n, m] = aB[:,:] + + ##============================= + ## Apply bfe to image + ##============================= + + ## pad and expand kernels + ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior + + # array_pad = self.saturate(image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 + array_pad = image.copy().array + saturation_array = np.ones_like(array_pad) * self.saturation_level + where_sat = np.where(array_pad > saturation_array) + array_pad[ where_sat ] = saturation_array[ where_sat ] + array_pad = array_pad[4:-4,4:-4] + array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array + + + dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] + + + ### run in sub grids to reduce memory + + ## pad and expand kernels + t = np.zeros((bin_size*n_sub, n_sub)) + for row in range(t.shape[0]): + t[row, row//(bin_size) ] =1 + + + + for gj in range(num_grids): + for gi in range(num_grids): + + a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) + + + for comp in range(4): + for j in range(2*nbfe+1): + for i in range(2*nbfe+1): + tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid + a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') + + #convolve aX_ij with Q_ij + for comp in range(4): + for dy in range(-nbfe, nbfe+1): + for dx in range(-nbfe, nbfe+1): + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ + *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] + + dj = int(np.sin(comp*np.pi/2)) + di = int(np.cos(comp*np.pi/2)) + + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ + array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) + + image.array[:,:] -= dQ_components.sum(axis=0) + image.array[:,1:] += dQ_components[0][:,:-1] + image.array[1:,:] += dQ_components[1][:-1,:] + image.array[:,:-1] += dQ_components[2][:,1:] + image.array[:-1,:] += dQ_components[3][1:,:] + + return image + + def apply(self, image): + image = self.model(image) + return image def set_diff(self, im=None): if self.save_diff: @@ -107,147 +299,147 @@ def qe(self, im): im.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array return im - def bfe(self, im): - """ - Apply brighter-fatter effect. - Brighter fatter effect is a non-linear effect that deflects photons due to the - the eletric field built by the accumulated charges. This effect exists in both - CCD and CMOS detectors and typically percent level change in charge. - The built-in electric field by the charges in pixels tends to repulse charges - to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. - This effect can also be understood effectly as change in pixel area and pixel - boundaries. - BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change - in the detector effect charaterization file. Kernel of the total pixel area, however, - is not sufficient. Image simulation of the brighter fatter effect requires the shift - of the four pixel boundaries. Before we get better data, we solve for the boundary - shift components from the kernel of total pixel area by assumming several symmetric constraints. - Input - im : Image - BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 - """ + # def bfe(self, im): + # """ + # Apply brighter-fatter effect. + # Brighter fatter effect is a non-linear effect that deflects photons due to the + # the eletric field built by the accumulated charges. This effect exists in both + # CCD and CMOS detectors and typically percent level change in charge. + # The built-in electric field by the charges in pixels tends to repulse charges + # to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. + # This effect can also be understood effectly as change in pixel area and pixel + # boundaries. + # BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change + # in the detector effect charaterization file. Kernel of the total pixel area, however, + # is not sufficient. Image simulation of the brighter fatter effect requires the shift + # of the four pixel boundaries. Before we get better data, we solve for the boundary + # shift components from the kernel of total pixel area by assumming several symmetric constraints. + # Input + # im : Image + # BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 + # """ - nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) - bin_size = 128 - n_max = 32 - m_max = 32 - num_grids = 4 - n_sub = n_max//num_grids - m_sub = m_max//num_grids + # nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) + # bin_size = 128 + # n_max = 32 + # m_max = 32 + # num_grids = 4 + # n_sub = n_max//num_grids + # m_sub = m_max//num_grids - ##======================================================================= - ## solve boundary shfit kernel aX components - ##======================================================================= - a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 - a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 + # ##======================================================================= + # ## solve boundary shfit kernel aX components + # ##======================================================================= + # a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 + # a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 - ##solve aR aT aL aB for each a - for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) - for m in range(m_max): - a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) + # ##solve aR aT aL aB for each a + # for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) + # for m in range(m_max): + # a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) - ## assume two parity symmetries - a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. + # ## assume two parity symmetries + # a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. - r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection - B = (a[2,2], a[3,2], a[2,3], a[3,3], - a[4,2], a[2,4], a[3,4], a[4,4] ) + # r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection + # B = (a[2,2], a[3,2], a[2,3], a[3,3], + # a[4,2], a[2,4], a[3,4], a[4,4] ) - A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], - [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], - [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], - [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], - [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], - [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], - [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], - [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) + # A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], + # [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], + # [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], + # [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], + # [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], + # [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], + # [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], + # [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) - s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] + # s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] - aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s3 , -s1 , s1 , s3 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) + # aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], + # [ 0. , -s6 , -s5 , s5 , s6 ], + # [ 0. , -s3 , -s1 , s1 , s3 ], + # [ 0. , -s6 , -s5 , s5 , s6 ], + # [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) - aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], - [ -s7 , -s6 , -s4 , -s6 , -s7 ], - [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], - [ r*s6 , s5 , s2 , s5 , r*s6 ], - [ s7 , s6 , s4 , s6 , s7 ],]) + # aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], + # [ -s7 , -s6 , -s4 , -s6 , -s7 ], + # [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], + # [ r*s6 , s5 , s2 , s5 , r*s6 ], + # [ s7 , s6 , s4 , s6 , s7 ],]) - aL = aR[::-1, ::-1] - aB = aT[::-1, ::-1] + # aL = aR[::-1, ::-1] + # aB = aT[::-1, ::-1] - a_components[0, :,:, n, m] = aR[:,:] - a_components[1, :,:, n, m] = aT[:,:] - a_components[2, :,:, n, m] = aL[:,:] - a_components[3, :,:, n, m] = aB[:,:] + # a_components[0, :,:, n, m] = aR[:,:] + # a_components[1, :,:, n, m] = aT[:,:] + # a_components[2, :,:, n, m] = aL[:,:] + # a_components[3, :,:, n, m] = aB[:,:] - ##============================= - ## Apply bfe to image - ##============================= + # ##============================= + # ## Apply bfe to image + # ##============================= - ## pad and expand kernels - ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior + # ## pad and expand kernels + # ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior - array_pad = self.saturate(im.copy()).array[4:-4,4:-4] # img of interest 4088x4088 - array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array + # array_pad = self.saturate(im.copy()).array[4:-4,4:-4] # img of interest 4088x4088 + # array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array - dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] + # dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] - ### run in sub grids to reduce memory + # ### run in sub grids to reduce memory - ## pad and expand kernels - t = np.zeros((bin_size*n_sub, n_sub)) - for row in range(t.shape[0]): - t[row, row//(bin_size) ] =1 + # ## pad and expand kernels + # t = np.zeros((bin_size*n_sub, n_sub)) + # for row in range(t.shape[0]): + # t[row, row//(bin_size) ] =1 - for gj in range(num_grids): - for gi in range(num_grids): + # for gj in range(num_grids): + # for gi in range(num_grids): - a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) + # a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) - for comp in range(4): - for j in range(2*nbfe+1): - for i in range(2*nbfe+1): - tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid - a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') + # for comp in range(4): + # for j in range(2*nbfe+1): + # for i in range(2*nbfe+1): + # tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid + # a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') - #convolve aX_ij with Q_ij - for comp in range(4): - for dy in range(-nbfe, nbfe+1): - for dx in range(-nbfe, nbfe+1): - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ - *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] + # #convolve aX_ij with Q_ij + # for comp in range(4): + # for dy in range(-nbfe, nbfe+1): + # for dx in range(-nbfe, nbfe+1): + # dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + # += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ + # *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] - dj = int(np.sin(comp*np.pi/2)) - di = int(np.cos(comp*np.pi/2)) + # dj = int(np.sin(comp*np.pi/2)) + # di = int(np.cos(comp*np.pi/2)) - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ - array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) + # dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + # *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ + # array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) - im.array[:,:] -= dQ_components.sum(axis=0) - im.array[:,1:] += dQ_components[0][:,:-1] - im.array[1:,:] += dQ_components[1][:-1,:] - im.array[:,:-1] += dQ_components[2][:,1:] - im.array[:-1,:] += dQ_components[3][1:,:] + # im.array[:,:] -= dQ_components.sum(axis=0) + # im.array[:,1:] += dQ_components[0][:,:-1] + # im.array[1:,:] += dQ_components[1][:-1,:] + # im.array[:,:-1] += dQ_components[2][:,1:] + # im.array[:-1,:] += dQ_components[3][1:,:] - return im + # return im def get_eff_sky_bg(self, pointing=None): """ @@ -522,12 +714,15 @@ def add_persistence(self, im, pointing=None): pointing = self.pointing # load the dithers of sky images that were simulated - dither_sca_array=np.loadtxt(self.params['image']['dither_from_file']).astype(int) + # dither_sca_array=np.loadtxt(self.params['image']['dither_from_file']).astype(int) # select adjacent exposures for the same sca (within 10*roman.exptime) - dither_list_selected = dither_sca_array[dither_sca_array[:,1] == pointing.sca, 0] - dither_list_selected = dither_list_selected[ np.abs(dither_list_selected - pointing.visit) < 10 ] - p_list = np.array([get_pointing(self.params, i, self.sca) for i in dither_list_selected]) + # dither_list_selected = dither_sca_array[dither_sca_array[:,1] == pointing.sca, 0] + # dither_list_selected = dither_list_selected[ np.abs(dither_list_selected - pointing.visit) < 10 ] + # p_list = np.array([get_pointing(self.params, i, self.sca) for i in dither_list_selected]) + + p_list = np.array([get_pointing(self.params, i, self.sca) for i in range(pointing.visit - 10, pointing.visit)]) + dt_list = np.array([(pointing.date - p.date).total_seconds() for p in p_list]) p_pers = p_list[ np.where((dt_list > 0) & (dt_list < pointing.exptime*10))] @@ -789,48 +984,48 @@ def add_bias(self, im): im.array[:,:] += bias return im - def finalize_sky_im(self,im, pointing=None): - """ - Finalize sky background for subtraction from final image. Add dark current, - convert to analog voltage, and quantize. - - Input - im : sky image - """ - - if pointing is None: - pointing = self.pointing - - if self.df is None: - im.quantize() - im += self.im_dark - im = self.saturate(im) - im += self.im_read - im = self.e_to_ADU(im) - im.quantize() - else: - - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) - im_pad = galsim.Image(bound_pad) - im_pad.array[4:-4, 4:-4] = im.array[:,:] - - im_pad = self.qe(im_pad) - im_pad = self.bfe(im_pad) - im_pad = self.add_persistence(im_pad, pointing) - im_pad.quantize() - im_pad += self.im_dark - im_pad = self.saturate(im_pad) - im_pad = self.nonlinearity(im_pad) - im_pad = self.interpix_cap(im_pad) - im_pad = self.deadpix(im_pad) - im_pad = self.vtpe(im_pad) - im_pad += self.im_read - im_pad = self.add_gain(im_pad) - im_pad = self.add_bias(im_pad) - im_pad.quantize() - # output 4088x4088 img in uint16 - im.array[:,:] = im_pad.array[4:-4, 4:-4] - im = galsim.Image(im, dtype=np.uint16) - - return im \ No newline at end of file + # def finalize_sky_im(self,im, pointing=None): + # """ + # Finalize sky background for subtraction from final image. Add dark current, + # convert to analog voltage, and quantize. + + # Input + # im : sky image + # """ + + # if pointing is None: + # pointing = self.pointing + + # if self.df is None: + # im.quantize() + # im += self.im_dark + # im = self.saturate(im) + # im += self.im_read + # im = self.e_to_ADU(im) + # im.quantize() + # else: + + # bound_pad = galsim.BoundsI( xmin=1, ymin=1, + # xmax=4096, ymax=4096) + # im_pad = galsim.Image(bound_pad) + # im_pad.array[4:-4, 4:-4] = im.array[:,:] + + # im_pad = self.qe(im_pad) + # im_pad = self.bfe(im_pad) + # im_pad = self.add_persistence(im_pad, pointing) + # im_pad.quantize() + # im_pad += self.im_dark + # im_pad = self.saturate(im_pad) + # im_pad = self.nonlinearity(im_pad) + # im_pad = self.interpix_cap(im_pad) + # im_pad = self.deadpix(im_pad) + # im_pad = self.vtpe(im_pad) + # im_pad += self.im_read + # im_pad = self.add_gain(im_pad) + # im_pad = self.add_bias(im_pad) + # im_pad.quantize() + # # output 4088x4088 img in uint16 + # im.array[:,:] = im_pad.array[4:-4, 4:-4] + # im = galsim.Image(im, dtype=np.uint16) + + # return im \ No newline at end of file diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index f9925bbc..e6e18a59 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -66,8 +66,10 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): 'ignore_noise' : bool, 'sca_filepath' : str, 'dither_from_file': str, - 'sca_filepath': str, - 'save_diff': bool + 'save_diff': bool, + + 'quantum_efficiency': str, + 'brighter_fatter': str, } params = galsim.config.GetAllParams(config, base, req=req, opt=opt, ignore=ignore + extra_ignore)[0] @@ -77,16 +79,21 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.mjd = params["mjd"] self.exptime = params["exptime"] - self.ignore_noise = params.get("ignore_noise", False) + self.ignore_noise = params.get('ignore_noise', False) + # self.exptime = params.get('exptime', roman.exptime) # Default is roman standard exposure time. - self.stray_light = params.get("stray_light", False) - self.thermal_background = params.get("thermal_background", False) - self.reciprocity_failure = params.get("reciprocity_failure", False) - self.dark_current = params.get("dark_current", False) - self.nonlinearity = params.get("nonlinearity", False) - self.ipc = params.get("ipc", False) - self.read_noise = params.get("read_noise", False) - self.sky_subtract = params.get("sky_subtract", False) + self.stray_light = params.get('stray_light', False) + self.thermal_background = params.get('thermal_background', False) + self.reciprocity_failure = params.get('reciprocity_failure', False) + self.dark_current = params.get('dark_current', False) + self.nonlinearity = params.get('nonlinearity', False) + self.ipc = params.get('ipc', False) + self.read_noise = params.get('read_noise', False) + self.sky_subtract = params.get('sky_subtract', False) + + # [TODO]TEST + self.qe = params.get('quantum_efficiency', None) + self.bfe = params.get('brighter_fatter', None) # If draw_method isn't in image field, it may be in stamp. Check. self.draw_method = params.get("draw_method", base.get("stamp", {}).get("draw_method", "auto")) @@ -116,8 +123,10 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): sca_filepath = self.sca_filepath) # If user hasn't overridden the bandpass to use, get the standard one. - if "bandpass" not in config: - base["bandpass"] = galsim.config.BuildBandpass(base["image"], "bandpass", base, logger=logger) + if 'bandpass' not in config: + base['bandpass'] = galsim.config.BuildBandpass(base['image'], 'bandpass', base, logger=logger) + + self.base = base return roman.n_pix, roman.n_pix @@ -271,11 +280,30 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) im_pad = galsim.Image(bound_pad) im_pad.array[4:-4, 4:-4] = image.array[:,:] self.effects.set_diff(im_pad) - im_pad = self.effects.qe(im_pad) - self.effects.diff('qe', im_pad) - - im_pad = self.effects.bfe(im_pad) - self.effects.diff('bfe', im_pad) + + # im_pad = self.effects.qe(im_pad) + # self.effects.diff('qe', im_pad) + if self.qe: + qe = self.effects.quantum_efficiency( + params=self.base, + logger=logger, + model=self.qe, + sca_filepath=self.sca_filepath + ) + im_pad = qe.apply(image = im_pad) + self.effects.diff('qe', im_pad) + + # im_pad = self.effects.bfe(im_pad) + # self.effects.diff('bfe', im_pad) + if self.bfe: + bfe = self.effects.bfe( + params=self.base, + logger=logger, + model=self.bfe, + sca_filepath=self.sca_filepath + ) + im_pad = bfe.apply(image = im_pad) + self.effects.diff('bfe', im_pad) im_pad = self.effects.add_persistence(im_pad) self.effects.diff('pers', im_pad) From 2532065242f18a45417ce87e77c21183f62e64c5 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 05:46:30 -0400 Subject: [PATCH 05/26] Start refactoring the detector effects in a subfolder --- config/was.yaml | 36 ++- roman_imsim/effects/__init__.py | 9 + roman_imsim/effects/background.py | 56 +++++ roman_imsim/effects/brighter_fatter.py | 172 +++++++++++++++ roman_imsim/effects/dark_current.py | 43 ++++ roman_imsim/effects/nonlinearity.py | 45 ++++ roman_imsim/effects/persistence.py | 130 +++++++++++ roman_imsim/effects/quantum_efficiency.py | 26 +++ roman_imsim/effects/recip_failure.py | 25 +++ roman_imsim/effects/roman_effects.py | 62 ++++++ roman_imsim/effects/saturate.py | 39 ++++ roman_imsim/effects/utils.py | 55 +++++ roman_imsim/sca.py | 255 +++++++++++++--------- 13 files changed, 835 insertions(+), 118 deletions(-) create mode 100644 roman_imsim/effects/__init__.py create mode 100644 roman_imsim/effects/background.py create mode 100644 roman_imsim/effects/brighter_fatter.py create mode 100644 roman_imsim/effects/dark_current.py create mode 100644 roman_imsim/effects/nonlinearity.py create mode 100644 roman_imsim/effects/persistence.py create mode 100644 roman_imsim/effects/quantum_efficiency.py create mode 100644 roman_imsim/effects/recip_failure.py create mode 100644 roman_imsim/effects/roman_effects.py create mode 100644 roman_imsim/effects/saturate.py create mode 100644 roman_imsim/effects/utils.py diff --git a/config/was.yaml b/config/was.yaml index 7f3ce2e7..b19daea9 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -88,17 +88,31 @@ image: # read_noise: False # sky_subtract: False - ignore_noise: False - quantum_efficiency: lab_model - brighter_fatter: lab_model - stray_light: True - thermal_background: True - reciprocity_failure: True - dark_current: True - nonlinearity: True - ipc: True - read_noise: True - sky_subtract: False + add_effects: + background: + model: simple_model + thermal_background: True + stray_light: True + quantum_efficiency: + model: lab_model + brighter_fatter: + model: lab_model + saturation_level: 100000 + persistence: + model: lab_model + # reciprocity_failure: + # model: simple_model + dark_current: + model: lab_model + saturate: + model: lab_model + nonlinearity: + model: lab_model + # ipc: + # model: simple_model + # read_noise: + # model: simple_model + # sky_subtract: False # dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt sca_filepath: /hpc/group/cosmology/phy-lsst/roman_sensors diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py new file mode 100644 index 00000000..2a2669f5 --- /dev/null +++ b/roman_imsim/effects/__init__.py @@ -0,0 +1,9 @@ +from .roman_effects import roman_effects +from .brighter_fatter import brighter_fatter +from .nonlinearity import nonlinearity +from .background import background +from .quantum_efficiency import quantum_efficiency +from .persistence import persistence +from .recip_failure import recip_failure +from .dark_current import dark_current +from .saturate import saturate \ No newline at end of file diff --git a/roman_imsim/effects/background.py b/roman_imsim/effects/background.py new file mode 100644 index 00000000..fdf16f03 --- /dev/null +++ b/roman_imsim/effects/background.py @@ -0,0 +1,56 @@ +import os +import numpy as np +import fitsio as fio +import galsim +from roman_imsim.effects import roman_effects +import galsim.roman as roman +from .utils import sca_number_to_file + +class background(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + self.thermal_background = self.params['thermal_background'] if 'thermal_background' in self.params else False + self.stray_light = self.params['stray_light'] if 'stray_light' in self.params else False + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + if self.save_diff: + orig = image.copy() + + self.logger.warning("Simple model will be applied for background.") + pointing = self.pointing + # Build current specification sky level if sky level not given + if self.force_cvz: + radec = self.translate_cvz(pointing.radec) + else: + radec = pointing.radec + sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) + self.logger.debug('Adding sky_level = %s',sky_level) + + if self.stray_light: + self.logger.debug('Stray light fraction = %s',roman.stray_light_fraction) + sky_level *= (1.0 + roman.stray_light_fraction) + # Create sky image + self.sky = galsim.Image(bounds=image.bounds, wcs=pointing.WCS) + pointing.WCS.makeSkyImage(self.sky, sky_level) + if self.thermal_background: + tb = roman.thermal_backgrounds[pointing.filter] * pointing.exptime + self.logger.debug('Adding thermal background: %s',tb) + self.sky += tb + self.sky.addNoise(self.noise) + + # [TODO] Not entirely sure about this block, since the 'auto' option is meant to let the software choose which drawing method to use based on the total flux. + if self.base['image']['draw_method'] not in ['phot', 'auto']: + image.addNoise(self.noise) + + # Adding sky level to the image. + image += self.sky[self.sky.bounds & image.bounds] + if self.save_diff: + prev = image.copy() + diff = prev - orig + diff.write(os.path.join(self.diff_dir, 'sky_a.fits')) + return image \ No newline at end of file diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/effects/brighter_fatter.py new file mode 100644 index 00000000..46355704 --- /dev/null +++ b/roman_imsim/effects/brighter_fatter.py @@ -0,0 +1,172 @@ +import os +import numpy as np +import fitsio as fio +from roman_imsim.effects import roman_effects +from .utils import sca_number_to_file + +class brighter_fatter(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.info("No bfe effect will be applied.") + return image + + def lab_model(self, image): + """ + Apply brighter-fatter effect. + Brighter fatter effect is a non-linear effect that deflects photons due to the + the eletric field built by the accumulated charges. This effect exists in both + CCD and CMOS detectors and typically percent level change in charge. + The built-in electric field by the charges in pixels tends to repulse charges + to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. + This effect can also be understood effectly as change in pixel area and pixel + boundaries. + BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change + in the detector effect charaterization file. Kernel of the total pixel area, however, + is not sufficient. Image simulation of the brighter fatter effect requires the shift + of the four pixel boundaries. Before we get better data, we solve for the boundary + shift components from the kernel of total pixel area by assumming several symmetric constraints. + Input + im : Image + BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 + """ + if self.sca_filepath is None: + self.logger.warning("No BFE kernel data file provided; no bfe effect will be applied.") + return image + + self.logger.warning("Lab measured model will be applied for brighter-fatter effect.") + + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) + bin_size = 128 + n_max = 32 + m_max = 32 + num_grids = 4 + n_sub = n_max//num_grids + m_sub = m_max//num_grids + + ##======================================================================= + ## solve boundary shfit kernel aX components + ##======================================================================= + a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 + a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 + + ##solve aR aT aL aB for each a + for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) + for m in range(m_max): + a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) + + ## assume two parity symmetries + a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. + + r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection + B = (a[2,2], a[3,2], a[2,3], a[3,3], + a[4,2], a[2,4], a[3,4], a[4,4] ) + + A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], + [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], + [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], + [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], + [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], + [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], + [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], + [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) + + + s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] + + aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s3 , -s1 , s1 , s3 ], + [ 0. , -s6 , -s5 , s5 , s6 ], + [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) + + + aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], + [ -s7 , -s6 , -s4 , -s6 , -s7 ], + [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], + [ r*s6 , s5 , s2 , s5 , r*s6 ], + [ s7 , s6 , s4 , s6 , s7 ],]) + + + aL = aR[::-1, ::-1] + aB = aT[::-1, ::-1] + + + + + a_components[0, :,:, n, m] = aR[:,:] + a_components[1, :,:, n, m] = aT[:,:] + a_components[2, :,:, n, m] = aL[:,:] + a_components[3, :,:, n, m] = aB[:,:] + + ##============================= + ## Apply bfe to image + ##============================= + + ## pad and expand kernels + ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior + + # array_pad = self.saturate(image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 + array_pad = image.copy().array + saturation_array = np.ones_like(array_pad) * self.saturation_level + where_sat = np.where(array_pad > saturation_array) + array_pad[ where_sat ] = saturation_array[ where_sat ] + array_pad = array_pad[4:-4,4:-4] + array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array + + + dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] + + + ### run in sub grids to reduce memory + + ## pad and expand kernels + t = np.zeros((bin_size*n_sub, n_sub)) + for row in range(t.shape[0]): + t[row, row//(bin_size) ] =1 + + + + for gj in range(num_grids): + for gi in range(num_grids): + + a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) + + + for comp in range(4): + for j in range(2*nbfe+1): + for i in range(2*nbfe+1): + tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid + a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') + + #convolve aX_ij with Q_ij + for comp in range(4): + for dy in range(-nbfe, nbfe+1): + for dx in range(-nbfe, nbfe+1): + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ + *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] + + dj = int(np.sin(comp*np.pi/2)) + di = int(np.cos(comp*np.pi/2)) + + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ + array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) + + image.array[:,:] -= dQ_components.sum(axis=0) + image.array[:,1:] += dQ_components[0][:,:-1] + image.array[1:,:] += dQ_components[1][:-1,:] + image.array[:,:-1] += dQ_components[2][:,1:] + image.array[:-1,:] += dQ_components[3][1:,:] + + return image \ No newline at end of file diff --git a/roman_imsim/effects/dark_current.py b/roman_imsim/effects/dark_current.py new file mode 100644 index 00000000..d8df88e5 --- /dev/null +++ b/roman_imsim/effects/dark_current.py @@ -0,0 +1,43 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from .utils import sca_number_to_file, get_pointing + +class dark_current(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("Simple model will be applied for dark current.") + exptime = self.pointing.exptime + self.dark_current_ = roman.dark_current * exptime + self.im_dark = image.copy() + dark_current_ = self.dark_current_ + dark_noise = galsim.DeviateNoise(galsim.PoissonDeviate(self.rng, dark_current_)) + image.addNoise(dark_noise) + self.im_dark = image - self.im_dark + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No dark current file provided; no dark current will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + exptime = self.pointing.exptime + self.logger.warning("Lab measured model will be applied for dark current.") + self.dark_current_ = roman.dark_current * exptime + self.df['DARK'][:, :].flatten() * exptime + dark_current_ = self.dark_current_.clip(0) + # opt for numpy random geneator instead for speed + self.im_dark = self.rng_np.poisson(dark_current_).reshape(image.array.shape).astype(image.dtype) + image.array[:,:] += self.im_dark + return image \ No newline at end of file diff --git a/roman_imsim/effects/nonlinearity.py b/roman_imsim/effects/nonlinearity.py new file mode 100644 index 00000000..dfa9359f --- /dev/null +++ b/roman_imsim/effects/nonlinearity.py @@ -0,0 +1,45 @@ +import os +import fitsio as fio +import galsim.roman as roman +from roman_imsim.effects import roman_effects +from .utils import sca_number_to_file + +class nonlinearity(roman_effects): + """ + Applying a quadratic non-linearity. + + Note that users who wish to apply some other nonlinearity function (perhaps for other NIR + detectors, or for CCDs) can use the more general nonlinearity routine, which uses the + following syntax: + final_image.applyNonlinearity(NLfunc=NLfunc) + with NLfunc being a callable function that specifies how the output image pixel values + should relate to the input ones. + """ + + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.info("Galsim.roman.NLfunc will be applied for simulating non-linearity effect.") + image.applyNonlinearity(NLfunc=roman.NLfunc) + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No non-linearity data file provided; no non-linearity effect will be applied.") + return image + + self.logger.warning("Lab measured model will be applied for non-linearity effect.") + + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + image.array[:,:] -= self.df['CNL'][0,:,:][0] * image.array**2 +\ + self.df['CNL'][1,:,:][0] * image.array**3 +\ + self.df['CNL'][2,:,:][0] * image.array**4 + + return image \ No newline at end of file diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/effects/persistence.py new file mode 100644 index 00000000..80852c2e --- /dev/null +++ b/roman_imsim/effects/persistence.py @@ -0,0 +1,130 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class persistence(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + p_list = np.array([get_pointing(self.base, i, self.sca) for i in range(self.visit - 10, self.visit)]) + dt_list = np.array([(self.pointing.date - p.date).total_seconds() for p in p_list]) + self.p_pers = p_list[ np.where((dt_list > 0) & (dt_list < self.pointing.exptime*10))] + + def simple_model(self, image): + for p in self.p_pers: + dt = (self.pointing.date - p.date).total_seconds() - self.pointing.exptime/2 ##avg time since end of exposures + # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] + # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] + imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) + fn = os.path.join(self.base['output']['dir'], imfilename) + + # [TODO] + if not os.path.exists(fn): + continue + + ## apply all the effects that occured before persistence on the previouse exposures + ## since max of the sky background is of order 100, it is thus negligible for persistence + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + x = galsim.Image(bound_pad) + x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] + + recip_failure_param = self.base['image']['add_effects']['recip_failure'] + recip_failure = effects.recip_failure(recip_failure_param, self.base, self.logger, self.rng) + x = recip_failure.apply(image = x) + + x.array.clip(0) ##remove negative stimulus + + image.array[:,:] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * self.pointing.exptime + + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No persistence data file provided; no persistence effect will be applied.") + return image + + self.logger.warning("Lab measured model will be applied for persistence effect.") + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + #setup parameters for persistence + Q01 = self.df['PERSIST'].read_header()['Q01'] + Q02 = self.df['PERSIST'].read_header()['Q02'] + Q03 = self.df['PERSIST'].read_header()['Q03'] + Q04 = self.df['PERSIST'].read_header()['Q04'] + Q05 = self.df['PERSIST'].read_header()['Q05'] + Q06 = self.df['PERSIST'].read_header()['Q06'] + alpha = self.df['PERSIST'].read_header()['ALPHA'] + + #iterate over previous exposures + for p in self.p_pers: + dt = (self.pointing.date - p.date).total_seconds() - self.pointing.exptime/2 ##avg time since end of exposures + fac_dt = (self.pointing.exptime/2.) / dt ##linear time dependence (approximate until we get t1 and Delat t of the data) + # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] + # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] + imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) + fn = os.path.join(self.base['output']['dir'], imfilename) + + # [TODO] + if not os.path.exists(fn): + continue + + ## apply all the effects that occured before persistence on the previouse exposures + ## since max of the sky background is of order 100, it is thus negligible for persistence + ## same for brighter fatter effect + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + x = galsim.Image(bound_pad) + x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] + # x = self.qe(x).array[:,:] + + qe_param = self.base['image']['add_effects']['quantum_efficiency'] + qe = effects.quantum_efficiency(qe_param, self.base, self.logger, self.rng) + x = qe.apply(image = x).array[:, :] + + x = x.clip(0.1) ##remove negative and zero stimulus + + ## Do linear interpolation + a = np.zeros(x.shape) + a += ((x < Q01)) * x/Q01 + a += ((x >= Q01) & (x < Q02)) * (Q02-x)/(Q02-Q01) + image.array[:,:] += a * self.df['PERSIST'][0,:,:][0] * fac_dt + + + a = np.zeros(x.shape) + a += ((x >= Q01) & (x < Q02)) * (x-Q01)/(Q02-Q01) + a += ((x >= Q02) & (x < Q03)) * (Q03-x)/(Q03-Q02) + image.array[:,:] += a * self.df['PERSIST'][1,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q02) & (x < Q03)) * (x-Q02)/(Q03-Q02) + a += ((x >= Q03) & (x < Q04)) * (Q04-x)/(Q04-Q03) + image.array[:,:] += a * self.df['PERSIST'][2,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q03) & (x < Q04)) * (x-Q03)/(Q04-Q03) + a += ((x >= Q04) & (x < Q05)) * (Q05-x)/(Q05-Q04) + image.array[:,:] += a * self.df['PERSIST'][3,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q04) & (x < Q05)) * (x-Q04)/(Q05-Q04) + a += ((x >= Q05) & (x < Q06)) * (Q06-x)/(Q06-Q05) + image.array[:,:] += a * self.df['PERSIST'][4,:,:][0] * fac_dt + + a = np.zeros(x.shape) + a += ((x >= Q05) & (x < Q06)) * (x-Q05)/(Q06-Q05) + a += ((x >= Q06)) * (x/Q06)**alpha ##avoid fractional power of negative values + image.array[:,:] += a * self.df['PERSIST'][5,:,:][0] * fac_dt + + return image \ No newline at end of file diff --git a/roman_imsim/effects/quantum_efficiency.py b/roman_imsim/effects/quantum_efficiency.py new file mode 100644 index 00000000..589e7003 --- /dev/null +++ b/roman_imsim/effects/quantum_efficiency.py @@ -0,0 +1,26 @@ +import os +import numpy as np +import fitsio as fio +import galsim +from roman_imsim.effects import roman_effects +import galsim.roman as roman +from .utils import sca_number_to_file + +class quantum_efficiency(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No QE data file provided; a default value of QE = 1 will be used.") + return image + + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + self.logger.warning("Lab measured model will be applied for quantum efficiency.") + image.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array + return image \ No newline at end of file diff --git a/roman_imsim/effects/recip_failure.py b/roman_imsim/effects/recip_failure.py new file mode 100644 index 00000000..4b731717 --- /dev/null +++ b/roman_imsim/effects/recip_failure.py @@ -0,0 +1,25 @@ +import os +import numpy as np +import fitsio as fio +import galsim +from roman_imsim.effects import roman_effects +import galsim.roman as roman +from .utils import sca_number_to_file + +class recip_failure(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.alpha = self.params['alpha'] if 'alpha' in self.params else roman.reciprocity_alpha + self.base_flux = self.params['base_flux'] if 'base_flux' in self.params else 1.0 + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + # Add reciprocity effect + exptime = self.pointing.exptime + image.addReciprocityFailure(exp_time=exptime, alpha=self.alpha, base_flux=self.base_flux) + return image \ No newline at end of file diff --git a/roman_imsim/effects/roman_effects.py b/roman_imsim/effects/roman_effects.py new file mode 100644 index 00000000..ebedf03c --- /dev/null +++ b/roman_imsim/effects/roman_effects.py @@ -0,0 +1,62 @@ +import os +import fitsio as fio +import galsim as galsim +import numpy as np +import galsim.roman as roman +from galsim.config import ParseValue +from .utils import get_pointing + +class roman_effects(object): + """ + Class to simulate non-idealities and noise of roman detector images. + """ + def __init__(self, params, base, logger, rng, rng_iter=None): + self.params = params + self.base = base + self.visit = int(self.base['input']['obseq_data']['visit']) + self.sca = base['image']['SCA'] + self.filter = base['image']['filter'] + self.sca_filepath = base['image']['sca_filepath'] + self.rng_iter = rng_iter if rng_iter else self.visit * self.sca + + self.rng = rng + self.noise = galsim.PoissonNoise(self.rng) + self.rng_np = np.random.default_rng(self.rng_iter) + self.pointing = get_pointing(self.base, self.visit, self.sca) + self.exptime = self.pointing.exptime + self.logger = logger + + self.force_cvz = False + if 'force_cvz' in self.base['image']['wcs']: + if self.base['image']['wcs']['force_cvz']: + self.force_cvz=True + + self.save_diff = False + if 'save_diff' in self.base['image']: + self.save_diff = bool(self.base['image']['save_diff']) + if 'diff_dir' in self.base['output']: + self.diff_dir = self.base['output']['diff_dir'] + else: + self.diff_dir = self.base['output']['dir'] + + def simple_model(self, image): + self.logger.info("Applying the default model...") + return image + + def apply(self, image): + image = self.model(image) + return image + + def set_diff(self, im=None): + if self.save_diff: + self.pre = im.copy() + self.pre.write('bg.fits', dir=self.diff_dir) + return + + def diff(self, msg, im=None, verbose=True): + if self.save_diff: + diff = im-self.pre + diff.write('%s_diff.fits'%msg , dir=self.diff_dir) + self.pre = im.copy() + im.write('%s_cumul.fits'%msg, dir=self.diff_dir) + return \ No newline at end of file diff --git a/roman_imsim/effects/saturate.py b/roman_imsim/effects/saturate.py new file mode 100644 index 00000000..47d82a3a --- /dev/null +++ b/roman_imsim/effects/saturate.py @@ -0,0 +1,39 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from .utils import sca_number_to_file, get_pointing + +class saturate(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 + + def simple_model(self, image): + self.logger.warning("Simple model will be applied for saturation.") + saturation_array = np.ones_like(image.array) * self.saturation_level + where_sat = np.where(image.array > saturation_array) + image.array[ where_sat ] = saturation_array[ where_sat ] + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No saturation data file provided; no saturation effect will be applied.") + return image + + self.logger.warning("Lab measured model will be applied for saturation effect.") + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + saturation_array = self.df['SATURATE'][:,:] #4096x4096 array + where_sat = np.where(image.array > saturation_array) + image.array[ where_sat ] = saturation_array[ where_sat ] + return image \ No newline at end of file diff --git a/roman_imsim/effects/utils.py b/roman_imsim/effects/utils.py new file mode 100644 index 00000000..fb3c9ac3 --- /dev/null +++ b/roman_imsim/effects/utils.py @@ -0,0 +1,55 @@ +import galsim as galsim +import galsim.roman as roman +from roman_imsim.obseq import ObSeqDataLoader + +sca_number_to_file = { + 1 : 'SCA_22066_211227_v001.fits', + 2 : 'SCA_21815_211221_v001.fits', + 3 : 'SCA_21946_211225_v001.fits', + 4 : 'SCA_22073_211229_v001.fits', + 5 : 'SCA_21816_211222_v001.fits', + 6 : 'SCA_20663_211102_v001.fits', + 7 : 'SCA_22069_211228_v001.fits', + 8 : 'SCA_21641_211216_v001.fits', + 9 : 'SCA_21813_211219_v001.fits', + 10 : 'SCA_22078_211230_v001.fits', + 11 : 'SCA_21947_211226_v001.fits', + 12 : 'SCA_22077_211230_v001.fits', + 13 : 'SCA_22067_211227_v001.fits', + 14 : 'SCA_21814_211220_v001.fits', + 15 : 'SCA_21645_211228_v001.fits', + 16 : 'SCA_21643_211218_v001.fits', + 17 : 'SCA_21319_211211_v001.fits', + 18 : 'SCA_20833_211116_v001.fits', + } + +class get_pointing(object): + """ + Class to store stuff about the telescope + """ + def __init__(self, params, visit, SCA): + + self.params = params + file_name = params['input']['obseq_data']['file_name'] + obseq_data = ObSeqDataLoader(file_name, visit, SCA, logger=None) + self.filter = obseq_data.ob['filter'] + self.sca = obseq_data.ob['sca'] + self.visit = obseq_data.ob['visit'] + self.date = obseq_data.ob['date'] + self.exptime = obseq_data.ob['exptime'] + self.bpass = roman.getBandpasses()[self.filter] + self.WCS = roman.getWCS(world_pos = galsim.CelestialCoord(ra=obseq_data.ob['ra'], \ + dec=obseq_data.ob['dec']), + PA = obseq_data.ob['pa'], + date = self.date, + SCAs = self.sca, + PA_is_FPA = True + )[self.sca] + self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2),int(roman.n_pix/2))) + +def translate_cvz(orig_radec, field_ra=9.5, field_dec=-44, cvz_ra=61.24, cvz_dec=-48.42): + ra = orig_radec.ra/galsim.degrees-field_ra + dec = orig_radec.dec/galsim.degrees-field_dec + ra += cvz_ra / np.cos(cvz_dec*np.pi/180) + dec += cvz_dec + return galsim.CelestialCoord(ra*galsim.degrees,dec*galsim.degrees) \ No newline at end of file diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index e6e18a59..8bcd00cc 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -10,6 +10,7 @@ import numpy as np from .detector_effects import detector_effects +import roman_imsim.effects as roman_effects class RomanSCAImageBuilder(ScatteredImageBuilder): @@ -59,7 +60,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): 'thermal_background' : bool, 'reciprocity_failure' : bool, 'dark_current' : bool, - 'nonlinearity' : bool, 'ipc' : bool, 'read_noise' : bool, 'sky_subtract' : bool, @@ -69,7 +69,11 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): 'save_diff': bool, 'quantum_efficiency': str, - 'brighter_fatter': str, + # 'brighter_fatter': str, + # 'nonlinearity' : str, + 'brighter_fatter': bool, + 'nonlinearity' : bool, + 'add_effects' : dict, } params = galsim.config.GetAllParams(config, base, req=req, opt=opt, ignore=ignore + extra_ignore)[0] @@ -86,14 +90,14 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.thermal_background = params.get('thermal_background', False) self.reciprocity_failure = params.get('reciprocity_failure', False) self.dark_current = params.get('dark_current', False) - self.nonlinearity = params.get('nonlinearity', False) self.ipc = params.get('ipc', False) self.read_noise = params.get('read_noise', False) self.sky_subtract = params.get('sky_subtract', False) # [TODO]TEST self.qe = params.get('quantum_efficiency', None) - self.bfe = params.get('brighter_fatter', None) + self.bfe = params.get('brighter_fatter', False) + self.nonlinearity = params.get('nonlinearity', False) # If draw_method isn't in image field, it may be in stamp. Check. self.draw_method = params.get("draw_method", base.get("stamp", {}).get("draw_method", "auto")) @@ -127,6 +131,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): base['bandpass'] = galsim.config.BuildBandpass(base['image'], 'bandpass', base, logger=logger) self.base = base + self.logger = logger return roman.n_pix, roman.n_pix @@ -223,6 +228,10 @@ def buildImage(self, config, base, image_num, obj_num, logger): logger.debug("image %d: Overlap = %s", image_num, str(bounds)) full_image[bounds] += stamps[k][bounds] stamps=None + + # [TODO] TEST + break + # # Bring the image so far up to a flat noise variance # current_var = FlattenNoiseVariance( @@ -254,7 +263,7 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) - self.effects.setup_sky(image, force_cvz=self.effects.force_cvz, stray_light=self.stray_light, thermal_background=self.thermal_background) + # self.effects.setup_sky(image, force_cvz=self.effects.force_cvz, stray_light=self.stray_light, thermal_background=self.thermal_background) # [TODO] quantize() at this step? # The image up to here is an expectation value. @@ -271,115 +280,147 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) # logger.debug("Adding poisson noise") # image += sky_image # image.addNoise(poisson_noise) - image = self.effects.add_background(image, draw_method=self.draw_method) - - if self.sca_filepath is not None: - ## create padded image - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) - im_pad = galsim.Image(bound_pad) - im_pad.array[4:-4, 4:-4] = image.array[:,:] - self.effects.set_diff(im_pad) + # image = self.effects.add_background(image, draw_method=self.draw_method) + + # create padded image + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + im_pad = galsim.Image(bound_pad) + im_pad.array[4:-4, 4:-4] = image.array[:,:] + + effects_list = self.base['image']['add_effects'].keys() + for effect_name in effects_list: + args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) + effect = getattr(roman_effects, effect_name)(*args) + im_pad = effect.apply(image = im_pad) - # im_pad = self.effects.qe(im_pad) - # self.effects.diff('qe', im_pad) - if self.qe: - qe = self.effects.quantum_efficiency( - params=self.base, - logger=logger, - model=self.qe, - sca_filepath=self.sca_filepath - ) - im_pad = qe.apply(image = im_pad) - self.effects.diff('qe', im_pad) - - # im_pad = self.effects.bfe(im_pad) - # self.effects.diff('bfe', im_pad) - if self.bfe: - bfe = self.effects.bfe( - params=self.base, - logger=logger, - model=self.bfe, - sca_filepath=self.sca_filepath - ) - im_pad = bfe.apply(image = im_pad) - self.effects.diff('bfe', im_pad) - - im_pad = self.effects.add_persistence(im_pad) - self.effects.diff('pers', im_pad) - - im_pad.quantize() - self.effects.diff('quantize1', im_pad) - - im_pad = self.effects.dark_current(im_pad) - self.effects.diff('dark', im_pad) + im_pad.quantize() + # output 4088x4088 img in uint16 + image.array[:,:] = im_pad.array[4:-4, 4:-4] + + # if self.sca_filepath is not None: + # ## create padded image + # bound_pad = galsim.BoundsI( xmin=1, ymin=1, + # xmax=4096, ymax=4096) + # im_pad = galsim.Image(bound_pad) + # im_pad.array[4:-4, 4:-4] = image.array[:,:] + # self.effects.set_diff(im_pad) - im_pad = self.effects.saturate(im_pad) - self.effects.diff('sat', im_pad) - - im_pad = self.effects.nonlinearity(im_pad) - self.effects.diff('cnl', im_pad) - - im_pad = self.effects.interpix_cap(im_pad) - self.effects.diff('ipc', im_pad) - - im_pad = self.effects.deadpix(im_pad) - self.effects.diff('deadpix', im_pad) - - im_pad = self.effects.vtpe(im_pad) - self.effects.diff('vtpe', im_pad) - - im_pad = self.effects.add_read_noise(im_pad) - self.effects.diff('read', im_pad) - - im_pad = self.effects.add_gain(im_pad) - self.effects.diff('gain', im_pad) - - im_pad = self.effects.add_bias(im_pad) - self.effects.diff('bias', im_pad) - - im_pad.quantize() - self.effects.diff('quantize2', im_pad) - - # output 4088x4088 img in uint16 - image.array[:,:] = im_pad.array[4:-4, 4:-4] - - # [TODO] - # # data quality image - # # 0x1 -> non-responsive - # # 0x2 -> hot pixel - # # 0x4 -> very hot pixel - # # 0x8 -> adjacent to pixel with strange response - # # 0x10 -> low CDS, high total noise pixel (may have strange settling behaviors, not recommended for precision applications) - # # 0x20 -> CNL fit went down to the minimum number of points (remaining degrees of freedom = 0) - # # 0x40 -> no solid-waffle solution for this region (set gain value to array median). normally occurs in a few small regions of some SCAs with lots of bad pixels. [recommend not to use these regions for WL analysis] - # # 0x80 -> wt==0 - # dq = self.df['BADPIX'][4:4092, 4:4092] - # # get weight map - # if wt is not None: - # dq[wt==0] += 128 + # # im_pad = self.effects.qe(im_pad) + # # self.effects.diff('qe', im_pad) + # if self.qe: + # qe = self.effects.quantum_efficiency( + # params=self.base, + # logger=logger, + # model=self.qe, + # sca_filepath=self.sca_filepath + # ) + # im_pad = qe.apply(image = im_pad) + # self.effects.diff('qe', im_pad) + + # # im_pad = self.effects.bfe(im_pad) + # # self.effects.diff('bfe', im_pad) + # if self.bfe: + # # bfe = self.effects.bfe( + # # params=self.base, + # # logger=logger, + # # model=self.bfe, + # # sca_filepath=self.sca_filepath + # # ) + # bfe = roman_effects.bfe( + # params = self.base['image']['add_effects']['brighter_fatter'], + # base = self.base, + # logger = self.logger, + # rng = self.rng, + # ) + # im_pad = bfe.apply(image = im_pad) + # self.effects.diff('bfe', im_pad) + + # im_pad = self.effects.add_persistence(im_pad) + # self.effects.diff('pers', im_pad) + + # im_pad.quantize() + # self.effects.diff('quantize1', im_pad) + + # im_pad = self.effects.dark_current(im_pad) + # self.effects.diff('dark', im_pad) + + # im_pad = self.effects.saturate(im_pad) + # self.effects.diff('sat', im_pad) - # sky_noise = self.sky.copy() - # sky_noise = self.finalize_sky_im(sky_noise, pointing) + # # im_pad = self.effects.nonlinearity(im_pad) + # # self.effects.diff('cnl', im_pad) + + # if self.nonlinearity: + # nonlinearity = roman_effects.nonlinearity( + # params = self.base['image']['add_effects']['nonlinearity'], + # base = self.base, + # logger = self.logger, + # rng = self.rng, + # ) + # im_pad = nonlinearity.apply(image = im_pad) + # self.effects.diff('cnl', im_pad) + + # im_pad = self.effects.interpix_cap(im_pad) + # self.effects.diff('ipc', im_pad) + + # im_pad = self.effects.deadpix(im_pad) + # self.effects.diff('deadpix', im_pad) + + # im_pad = self.effects.vtpe(im_pad) + # self.effects.diff('vtpe', im_pad) + + # im_pad = self.effects.add_read_noise(im_pad) + # self.effects.diff('read', im_pad) + + # im_pad = self.effects.add_gain(im_pad) + # self.effects.diff('gain', im_pad) + + # im_pad = self.effects.add_bias(im_pad) + # self.effects.diff('bias', im_pad) + + # im_pad.quantize() + # self.effects.diff('quantize2', im_pad) + + # # output 4088x4088 img in uint16 + # image.array[:,:] = im_pad.array[4:-4, 4:-4] + + # # [TODO] + # # # data quality image + # # # 0x1 -> non-responsive + # # # 0x2 -> hot pixel + # # # 0x4 -> very hot pixel + # # # 0x8 -> adjacent to pixel with strange response + # # # 0x10 -> low CDS, high total noise pixel (may have strange settling behaviors, not recommended for precision applications) + # # # 0x20 -> CNL fit went down to the minimum number of points (remaining degrees of freedom = 0) + # # # 0x40 -> no solid-waffle solution for this region (set gain value to array median). normally occurs in a few small regions of some SCAs with lots of bad pixels. [recommend not to use these regions for WL analysis] + # # # 0x80 -> wt==0 + # # dq = self.df['BADPIX'][4:4092, 4:4092] + # # # get weight map + # # if wt is not None: + # # dq[wt==0] += 128 + + # # sky_noise = self.sky.copy() + # # sky_noise = self.finalize_sky_im(sky_noise, pointing) - else: - image = self.effects.recip_failure(image) # Introduce reciprocity failure to image - image.quantize() # At this point in the image generation process, an integer number of photons gets detected - image = self.effects.dark_current(image) # Add dark current to image - image = self.effects.add_persistence(image) - image = self.effects.saturate(image) - image= self.effects.nonlinearity(image) # Apply nonlinearity - image = self.effects.interpix_cap(image) # Introduce interpixel capacitance to image. - image = self.effects.add_read_noise(image) - image = self.effects.e_to_ADU(image) # Convert electrons to ADU + # else: + # image = self.effects.recip_failure(image) # Introduce reciprocity failure to image + # image.quantize() # At this point in the image generation process, an integer number of photons gets detected + # image = self.effects.dark_current(image) # Add dark current to image + # image = self.effects.add_persistence(image) + # image = self.effects.saturate(image) + # image= self.effects.nonlinearity(image) # Apply nonlinearity + # image = self.effects.interpix_cap(image) # Introduce interpixel capacitance to image. + # image = self.effects.add_read_noise(image) + # image = self.effects.e_to_ADU(image) # Convert electrons to ADU # Make integer ADU now. - image.quantize() + # image.quantize() - if self.sky_subtract: - logger.debug("Subtracting sky image") - sky_image = self.effects.finalize_sky_im(self.effects.sky.copy()) - image -= sky_image + # if self.sky_subtract: + # logger.debug("Subtracting sky image") + # sky_image = self.effects.finalize_sky_im(self.effects.sky.copy()) + # image -= sky_image # Register this as a valid type From 1f291cd65fd995319555b7e03a654c7672e67472 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 05:48:31 -0400 Subject: [PATCH 06/26] remove debugging code --- roman_imsim/sca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 8bcd00cc..7a902903 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -230,7 +230,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): stamps=None # [TODO] TEST - break + # break # # Bring the image so far up to a flat noise variance From 23305b1a300d43e00033372a34431cc7cd0d3624 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 16:40:50 -0400 Subject: [PATCH 07/26] Add cross_refer method to roman_effects for internal cross-referencing within its child classes. --- config/was.yaml | 1 - roman_imsim/effects/brighter_fatter.py | 15 ++++++++------- roman_imsim/effects/persistence.py | 10 ++++++---- roman_imsim/effects/roman_effects.py | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+), 12 deletions(-) diff --git a/config/was.yaml b/config/was.yaml index b19daea9..eafbed60 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -97,7 +97,6 @@ image: model: lab_model brighter_fatter: model: lab_model - saturation_level: 100000 persistence: model: lab_model # reciprocity_failure: diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/effects/brighter_fatter.py index 46355704..3fbaef38 100644 --- a/roman_imsim/effects/brighter_fatter.py +++ b/roman_imsim/effects/brighter_fatter.py @@ -7,7 +7,7 @@ class brighter_fatter(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 + # self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 self.model = getattr(self, self.params['model']) if self.model is None: @@ -115,12 +115,13 @@ def lab_model(self, image): ## pad and expand kernels ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior - # array_pad = self.saturate(image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 - array_pad = image.copy().array - saturation_array = np.ones_like(array_pad) * self.saturation_level - where_sat = np.where(array_pad > saturation_array) - array_pad[ where_sat ] = saturation_array[ where_sat ] - array_pad = array_pad[4:-4,4:-4] + # array_pad = image.copy().array + # saturation_array = np.ones_like(array_pad) * self.saturation_level + # where_sat = np.where(array_pad > saturation_array) + # array_pad[ where_sat ] = saturation_array[ where_sat ] + # array_pad = array_pad[4:-4,4:-4] + saturate = self.cross_refer('saturate') + array_pad = saturate.apply(image = image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/effects/persistence.py index 80852c2e..f97c70db 100644 --- a/roman_imsim/effects/persistence.py +++ b/roman_imsim/effects/persistence.py @@ -22,6 +22,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): self.p_pers = p_list[ np.where((dt_list > 0) & (dt_list < self.pointing.exptime*10))] def simple_model(self, image): + self.logger.warning("Simple model will be applied for persistence effect.") for p in self.p_pers: dt = (self.pointing.date - p.date).total_seconds() - self.pointing.exptime/2 ##avg time since end of exposures # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] @@ -40,8 +41,9 @@ def simple_model(self, image): x = galsim.Image(bound_pad) x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] - recip_failure_param = self.base['image']['add_effects']['recip_failure'] - recip_failure = effects.recip_failure(recip_failure_param, self.base, self.logger, self.rng) + # recip_failure_param = self.base['image']['add_effects']['recip_failure'] + # recip_failure = effects.recip_failure(recip_failure_param, self.base, self.logger, self.rng) + recip_failure = self.cross_refer('recip_failure') x = recip_failure.apply(image = x) x.array.clip(0) ##remove negative stimulus @@ -73,6 +75,7 @@ def lab_model(self, image): fac_dt = (self.pointing.exptime/2.) / dt ##linear time dependence (approximate until we get t1 and Delat t of the data) # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] + imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) fn = os.path.join(self.base['output']['dir'], imfilename) @@ -89,8 +92,7 @@ def lab_model(self, image): x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] # x = self.qe(x).array[:,:] - qe_param = self.base['image']['add_effects']['quantum_efficiency'] - qe = effects.quantum_efficiency(qe_param, self.base, self.logger, self.rng) + qe = self.cross_refer('quantum_efficiency') x = qe.apply(image = x).array[:, :] x = x.clip(0.1) ##remove negative and zero stimulus diff --git a/roman_imsim/effects/roman_effects.py b/roman_imsim/effects/roman_effects.py index ebedf03c..e86f01b7 100644 --- a/roman_imsim/effects/roman_effects.py +++ b/roman_imsim/effects/roman_effects.py @@ -5,6 +5,7 @@ import galsim.roman as roman from galsim.config import ParseValue from .utils import get_pointing +import roman_imsim.effects as effects class roman_effects(object): """ @@ -43,6 +44,24 @@ def simple_model(self, image): self.logger.info("Applying the default model...") return image + def cross_refer(self, effect_name): + if effect_name not in self.base['image']['add_effects']: + try: + effect = getattr(effects, effect_name)({'model':'simple_model'}, self.base, self.logger, self.rng) + except Exception as e: + self.logger.warning(e) + # self.logger.warning("Effect %s is not implemented!"%(effect_name)) + return None + else: + try: + params = self.base['image']['add_effects'][effect_name] + effect = getattr(effects, effect_name)(params, self.base, self.logger, self.rng) + except Exception as e: + self.logger.warning(e) + # self.logger.warning("Effect %s is not implemented!"%(effect_name)) + return None + return effect + def apply(self, image): image = self.model(image) return image From c5607267206023434b3ca9f6fc369c6b0e4fcd6d Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 17:21:11 -0400 Subject: [PATCH 08/26] add some logging to bfe and recip_failure --- roman_imsim/effects/brighter_fatter.py | 5 ++--- roman_imsim/effects/recip_failure.py | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/effects/brighter_fatter.py index 3fbaef38..1a8d2285 100644 --- a/roman_imsim/effects/brighter_fatter.py +++ b/roman_imsim/effects/brighter_fatter.py @@ -15,7 +15,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): self.model = self.simple_model def simple_model(self, image): - self.logger.info("No bfe effect will be applied.") + self.logger.warning("No bfe effect will be applied.") return image def lab_model(self, image): @@ -40,11 +40,10 @@ def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No BFE kernel data file provided; no bfe effect will be applied.") return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for brighter-fatter effect.") - self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) bin_size = 128 n_max = 32 diff --git a/roman_imsim/effects/recip_failure.py b/roman_imsim/effects/recip_failure.py index 4b731717..aecffa53 100644 --- a/roman_imsim/effects/recip_failure.py +++ b/roman_imsim/effects/recip_failure.py @@ -20,6 +20,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): def simple_model(self, image): # Add reciprocity effect + self.logger.warning("Simple model will be applied for reciprocity failure effect.") exptime = self.pointing.exptime image.addReciprocityFailure(exp_time=exptime, alpha=self.alpha, base_flux=self.base_flux) return image \ No newline at end of file From 56e49624941732684629adb4d63077a3f20dead1 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 17:23:32 -0400 Subject: [PATCH 09/26] add dead_pix, interpix_cap, read_noise, vtpe --- roman_imsim/effects/__init__.py | 6 ++- roman_imsim/effects/dead_pix.py | 34 +++++++++++++++ roman_imsim/effects/interpix_cap.py | 65 +++++++++++++++++++++++++++++ roman_imsim/effects/persistence.py | 4 -- roman_imsim/effects/read_noise.py | 38 +++++++++++++++++ roman_imsim/effects/vtpe.py | 61 +++++++++++++++++++++++++++ 6 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 roman_imsim/effects/dead_pix.py create mode 100644 roman_imsim/effects/interpix_cap.py create mode 100644 roman_imsim/effects/read_noise.py create mode 100644 roman_imsim/effects/vtpe.py diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py index 2a2669f5..a656cad2 100644 --- a/roman_imsim/effects/__init__.py +++ b/roman_imsim/effects/__init__.py @@ -6,4 +6,8 @@ from .persistence import persistence from .recip_failure import recip_failure from .dark_current import dark_current -from .saturate import saturate \ No newline at end of file +from .saturate import saturate +from .interpix_cap import interpix_cap +from .dead_pix import dead_pix +from .vtpe import vtpe +from .read_noise import read_noise \ No newline at end of file diff --git a/roman_imsim/effects/dead_pix.py b/roman_imsim/effects/dead_pix.py new file mode 100644 index 00000000..88eddeaf --- /dev/null +++ b/roman_imsim/effects/dead_pix.py @@ -0,0 +1,34 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class dead_pix(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("No dead pixel will be applied.") + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No bad pixel data file provided; no dead pixel will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for dead pixel.") + dead_mask = self.df['BADPIX'][:,:]&1 #4096x4096 array + image.array[ dead_mask>0 ]=0 + + return image \ No newline at end of file diff --git a/roman_imsim/effects/interpix_cap.py b/roman_imsim/effects/interpix_cap.py new file mode 100644 index 00000000..2fc91eb9 --- /dev/null +++ b/roman_imsim/effects/interpix_cap.py @@ -0,0 +1,65 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class interpix_cap(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("Simple model will be applied for IPC effect.") + kernel=roman.ipc_kernel + image.applyIPC(kernel, edge_treatment='extend', fill_value=None) + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No IPC kernel data file provided; no IPC effect will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for IPC effect.") + # pad the array by one pixel at the four edges + num_grids = 4 ### num_grids <= 8 + grid_size = 4096//num_grids + + array_pad = image.array[4:-4,4:-4] #it's an array instead of img + array_pad = np.pad(array_pad, [(5,5),(5,5)], mode='symmetric') #4098x4098 array + + K = self.df['IPC'][:, :, :, :] ##3,3,512, 512 + + t = np.zeros((grid_size, 512)) + for row in range(t.shape[0]): + t[row, row//( grid_size//512) ] =1 + + array_out = np.zeros( (4096, 4096)) + ##split job in sub_grids to reduce memory + for gj in range(num_grids): + for gi in range(num_grids): + K_pad = np.zeros( (3,3, grid_size+2, grid_size+2) ) + + for j in range(3): + for i in range(3): + tmp = (t.dot(K[j,i,:,:])).dot(t.T) #grid_sizexgrid_size + K_pad[j,i,:,:] = np.pad(tmp, [(1,1),(1,1)], mode='symmetric') + + for dy in range(-1, 2): + for dx in range(-1,2): + + array_out[ gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ + +=K_pad[ 1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size ] \ + *array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] + + image.array[:,:] = array_out + return image \ No newline at end of file diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/effects/persistence.py index f97c70db..f4620e5c 100644 --- a/roman_imsim/effects/persistence.py +++ b/roman_imsim/effects/persistence.py @@ -2,9 +2,7 @@ import numpy as np import fitsio as fio import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects from galsim.config import ParseValue from .utils import sca_number_to_file, get_pointing @@ -41,8 +39,6 @@ def simple_model(self, image): x = galsim.Image(bound_pad) x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] - # recip_failure_param = self.base['image']['add_effects']['recip_failure'] - # recip_failure = effects.recip_failure(recip_failure_param, self.base, self.logger, self.rng) recip_failure = self.cross_refer('recip_failure') x = recip_failure.apply(image = x) diff --git a/roman_imsim/effects/read_noise.py b/roman_imsim/effects/read_noise.py new file mode 100644 index 00000000..797544aa --- /dev/null +++ b/roman_imsim/effects/read_noise.py @@ -0,0 +1,38 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class read_noise(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("Simple model will be applied for read noise.") + self.im_read = image.copy() + image.addNoise(self.read_noise) + self.im_read = image - self.im_read + # self.sky.addNoise(self.read_noise) + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No read noise data file provided; no read noise will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for read noise.") + rdn = self.df['READ'][2,:,:].flatten() #flattened 4096x4096 array + self.im_read = self.rng_np.normal(loc=0., scale=rdn).reshape(image.array.shape).astype(image.dtype) + image.array[:,:] += self.im_read + return image \ No newline at end of file diff --git a/roman_imsim/effects/vtpe.py b/roman_imsim/effects/vtpe.py new file mode 100644 index 00000000..0b5af82e --- /dev/null +++ b/roman_imsim/effects/vtpe.py @@ -0,0 +1,61 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class vtpe(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("No vertical trailing pixel effect will be applied.") + return image + + def lab_model(self, image): + """ + Apply vertical trailing pixel effect. + The vertical trailing pixel effect (VTPE) is a non-linear effect that is + related to readout patterns. + Q'[j,i] = Q[j,i] + f( Q[j,i] - Q[j-1, i] ), + where f( dQ ) = dQ ( a + b * ln(1 + |dQ|/dQ0) ) + Input + im : image + VTPE[0,512,512] : coefficient a binned in 8x8 + VTPE[1,512,512] : coefficient a + VTPE[2,512,512] : coefficient dQ0 + """ + + if self.sca_filepath is None: + self.logger.warning("No VTPE data file provided; no VTPE will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for VTPE.") + + # expand 512x512 arrays to 4096x4096 + t = np.zeros((4096, 512)) + for row in range(t.shape[0]): + t[row, row//8] =1 + a_vtpe = t.dot(self.df['VTPE'][0,:,:][0]).dot(t.T) + ## NaN check + if np.isnan(a_vtpe).any(): + self.logger.warning("vtpe skipped due to NaN in file") + return image + b_vtpe = t.dot(self.df['VTPE'][1,:,:][0]).dot(t.T) + dQ0 = t.dot(self.df['VTPE'][2,:,:][0]).dot(t.T) + + dQ = image.array - np.roll(image.array, 1, axis=0) + dQ[0,:] *= 0 + + image.array[:,:] += dQ * ( a_vtpe + b_vtpe * np.log( 1. + np.abs(dQ)/dQ0 )) + return image \ No newline at end of file From 52d3cd757656c537d94b500fff4cc502ec213cae Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 8 Jul 2025 17:56:51 -0400 Subject: [PATCH 10/26] add gain and bias model to effects --- config/was.yaml | 18 ++++++++++----- roman_imsim/effects/__init__.py | 4 +++- roman_imsim/effects/bias.py | 34 ++++++++++++++++++++++++++++ roman_imsim/effects/gain.py | 40 +++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 roman_imsim/effects/bias.py create mode 100644 roman_imsim/effects/gain.py diff --git a/config/was.yaml b/config/was.yaml index eafbed60..cf3f1751 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -99,7 +99,7 @@ image: model: lab_model persistence: model: lab_model - # reciprocity_failure: + # reciprocity_failure: # model: simple_model dark_current: model: lab_model @@ -107,10 +107,18 @@ image: model: lab_model nonlinearity: model: lab_model - # ipc: - # model: simple_model - # read_noise: - # model: simple_model + interpix_cap: + model: lab_model + dead_pix: + model: lab_model + vtpe: + model: lab_model + read_noise: + model: lab_model + gain: + model: lab_model + bias: + model: lab_model # sky_subtract: False # dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py index a656cad2..f664bc27 100644 --- a/roman_imsim/effects/__init__.py +++ b/roman_imsim/effects/__init__.py @@ -10,4 +10,6 @@ from .interpix_cap import interpix_cap from .dead_pix import dead_pix from .vtpe import vtpe -from .read_noise import read_noise \ No newline at end of file +from .read_noise import read_noise +from .gain import gain +from .bias import bias \ No newline at end of file diff --git a/roman_imsim/effects/bias.py b/roman_imsim/effects/bias.py new file mode 100644 index 00000000..941ad24a --- /dev/null +++ b/roman_imsim/effects/bias.py @@ -0,0 +1,34 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class bias(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("No bias will be applied.") + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No bias data provided; no bias will be applied.") + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for bias.") + bias = self.df['BIAS'][:,:] #4096x4096 img + + image.array[:,:] += bias + return image \ No newline at end of file diff --git a/roman_imsim/effects/gain.py b/roman_imsim/effects/gain.py new file mode 100644 index 00000000..2dbe0884 --- /dev/null +++ b/roman_imsim/effects/gain.py @@ -0,0 +1,40 @@ +import os +import numpy as np +import fitsio as fio +import galsim +import galsim.roman as roman +from roman_imsim.effects import roman_effects +import roman_imsim.effects as effects +from galsim.config import ParseValue +from .utils import sca_number_to_file, get_pointing + +class gain(roman_effects): + def __init__(self, params, base, logger, rng, rng_iter=None): + super().__init__(params, base, logger, rng, rng_iter) + + self.model = getattr(self, self.params['model']) + if self.model is None: + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.model = self.simple_model + + def simple_model(self, image): + self.logger.warning("Simple model will be applied for gain.") + image /= roman.gain + return image + + def lab_model(self, image): + if self.sca_filepath is None: + self.logger.warning("No gain data provided; galsim.roman.gain will be applied.") + image /= roman.gain + return image + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) + + self.logger.warning("Lab measured model will be applied for gain.") + gain_map = self.df['GAIN'][:, :] #32x32 img + + t = np.zeros((4096, 32)) + for row in range(t.shape[0]): + t[row, row//128] =1 + gain_expand = (t.dot(gain_map)).dot(t.T) #4096x4096 gain img + image.array[:,:] /= gain_expand + return image \ No newline at end of file From b0a8713c5be2155c036392ef5bf6f30cf9fbe924 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Fri, 11 Jul 2025 04:15:47 -0400 Subject: [PATCH 11/26] add setup_sky class for generating sky as well as sky subtracted images --- roman_imsim/effects/__init__.py | 3 ++- roman_imsim/effects/setup_sky.py | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 roman_imsim/effects/setup_sky.py diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py index f664bc27..f91a8d6d 100644 --- a/roman_imsim/effects/__init__.py +++ b/roman_imsim/effects/__init__.py @@ -12,4 +12,5 @@ from .vtpe import vtpe from .read_noise import read_noise from .gain import gain -from .bias import bias \ No newline at end of file +from .bias import bias +from .setup_sky import setup_sky \ No newline at end of file diff --git a/roman_imsim/effects/setup_sky.py b/roman_imsim/effects/setup_sky.py new file mode 100644 index 00000000..0e128518 --- /dev/null +++ b/roman_imsim/effects/setup_sky.py @@ -0,0 +1,46 @@ +import numpy as np +import galsim +import roman_imsim.effects as effects +from .utils import sca_number_to_file, get_pointing + +class setup_sky(object): + def __init__(self, base, logger, rng, rng_iter=None): + self.base = base + self.logger = logger + self.rng = rng + + self.visit = int(self.base['input']['obseq_data']['visit']) + self.sca = base['image']['SCA'] + self.pointing = get_pointing(self.base, self.visit, self.sca) + + def get_sky_image(self): + bounds = galsim.BoundsI( xmin=1, ymin=1, + xmax=4088, ymax=4088) + self.sky_img = galsim.Image(bounds=bounds, wcs=self.pointing.WCS) + self.pointing.WCS.makeSkyImage(self.sky_img, 0.) + + bound_pad = galsim.BoundsI( xmin=1, ymin=1, + xmax=4096, ymax=4096) + im_pad = galsim.Image(bound_pad) + im_pad.array[4:-4, 4:-4] = self.sky_img.array[:,:] + + effects_list = list(self.base['image']['add_effects']) + if 'background' not in effects_list: + return self.sky_img + + bkg_idx = effects_list.index('background') + for i in range(bkg_idx, len(effects_list)): + effect_name = effects_list[i] + args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) + effect = getattr(effects, effect_name)(*args) + im_pad = effect.apply(image = im_pad) + + im_pad.quantize() + # output 4088x4088 img in uint16 + self.sky_img = im_pad.array[4:-4, 4:-4] + self.sky_img = galsim.Image(self.sky_img, dtype=np.uint16) + + return self.sky_img + + def save_sky_img(self, outdir='.', sky_img_name='sky_img.fits'): + self.sky_img.write(sky_img_name, dir=outdir) \ No newline at end of file From cc56da01e3c1f5f2fa8aa4c4325c8b6f9b35a3e2 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Fri, 11 Jul 2025 04:20:01 -0400 Subject: [PATCH 12/26] modify config and sca.py for generating sky subtracted images --- config/was.yaml | 2 +- roman_imsim/sca.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/was.yaml b/config/was.yaml index cf3f1751..879d90dd 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -119,7 +119,7 @@ image: model: lab_model bias: model: lab_model - # sky_subtract: False + sky_subtract: False # dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt sca_filepath: /hpc/group/cosmology/phy-lsst/roman_sensors diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 7a902903..020f06c2 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -417,10 +417,13 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) # Make integer ADU now. # image.quantize() - # if self.sky_subtract: - # logger.debug("Subtracting sky image") - # sky_image = self.effects.finalize_sky_im(self.effects.sky.copy()) - # image -= sky_image + if self.sky_subtract: + logger.debug("Subtracting sky image") + sky = roman_effects.setup_sky(self.base, self.logger, self.rng) + sky_image = sky.get_sky_image() + image -= sky_image + sky.save_sky_img(outdir=self.base['output']['dir']) + # Register this as a valid type From 12b1c1e8ae9e7eb942e21ee28ff9db5f9a7ecd80 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 16 Oct 2024 18:02:47 -0400 Subject: [PATCH 13/26] rearrange functions in detector_physics.py to sca.py and detector_effects.py --- roman_imsim/sca.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 020f06c2..4a9d6673 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -9,6 +9,7 @@ from astropy.time import Time import numpy as np + from .detector_effects import detector_effects import roman_imsim.effects as roman_effects @@ -228,10 +229,9 @@ def buildImage(self, config, base, image_num, obj_num, logger): logger.debug("image %d: Overlap = %s", image_num, str(bounds)) full_image[bounds] += stamps[k][bounds] stamps=None - - # [TODO] TEST + + # # [TODO] # break - # # Bring the image so far up to a flat noise variance # current_var = FlattenNoiseVariance( From c3b6474af88ea636cdbb37b91030429957a6996b Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 29 Jul 2025 16:16:54 -0400 Subject: [PATCH 14/26] add .gitignore file --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..410edf52 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/build/* +/dist/* +/roman_imsim.egg-info/* \ No newline at end of file From bab72728ec80951109a0296d0b08c51e7f515d16 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 16 Oct 2024 18:02:47 -0400 Subject: [PATCH 15/26] rearrange functions in detector_physics.py to sca.py and detector_effects.py --- roman_imsim/sca.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 4a9d6673..65003019 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -8,6 +8,8 @@ from galsim.image import Image from astropy.time import Time import numpy as np +from astropy.time import Time +import numpy as np from .detector_effects import detector_effects From 212c0bf9f6cf86b508ae8306a14165b1661f11a8 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 16 Oct 2024 18:02:47 -0400 Subject: [PATCH 16/26] rearrange functions in detector_physics.py to sca.py and detector_effects.py --- roman_imsim/sca.py | 1 + 1 file changed, 1 insertion(+) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 65003019..6159ae36 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -12,6 +12,7 @@ import numpy as np + from .detector_effects import detector_effects import roman_imsim.effects as roman_effects From 98ff4c32d4009c7c894d44f5fbf12c621820dfe9 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 29 Jul 2025 17:56:59 -0400 Subject: [PATCH 17/26] reformat to meet pep8 --- roman_imsim/__init__.py | 4 +- roman_imsim/detector_effects.py | 1031 --------------------- roman_imsim/effects/background.py | 33 +- roman_imsim/effects/bias.py | 23 +- roman_imsim/effects/brighter_fatter.py | 179 ++-- roman_imsim/effects/dark_current.py | 20 +- roman_imsim/effects/dead_pix.py | 25 +- roman_imsim/effects/gain.py | 25 +- roman_imsim/effects/interpix_cap.py | 52 +- roman_imsim/effects/nonlinearity.py | 29 +- roman_imsim/effects/persistence.py | 92 +- roman_imsim/effects/quantum_efficiency.py | 17 +- roman_imsim/effects/read_noise.py | 23 +- roman_imsim/effects/recip_failure.py | 17 +- roman_imsim/effects/roman_effects.py | 53 +- roman_imsim/effects/saturate.py | 32 +- roman_imsim/effects/setup_sky.py | 35 +- roman_imsim/effects/utils.py | 60 +- roman_imsim/effects/vtpe.py | 30 +- roman_imsim/sca.py | 188 +--- 20 files changed, 390 insertions(+), 1578 deletions(-) delete mode 100644 roman_imsim/detector_effects.py diff --git a/roman_imsim/__init__.py b/roman_imsim/__init__.py index a7919744..dfba337a 100644 --- a/roman_imsim/__init__.py +++ b/roman_imsim/__init__.py @@ -18,6 +18,4 @@ from .wcs import * from .skycat import * from .photonOps import * -from .bandpass import * -# from .detector_physics import * -from .detector_effects import * \ No newline at end of file +from .bandpass import * \ No newline at end of file diff --git a/roman_imsim/detector_effects.py b/roman_imsim/detector_effects.py deleted file mode 100644 index 9bc12807..00000000 --- a/roman_imsim/detector_effects.py +++ /dev/null @@ -1,1031 +0,0 @@ -sca_number_to_file = { - 1 : 'SCA_22066_211227_v001.fits', - 2 : 'SCA_21815_211221_v001.fits', - 3 : 'SCA_21946_211225_v001.fits', - 4 : 'SCA_22073_211229_v001.fits', - 5 : 'SCA_21816_211222_v001.fits', - 6 : 'SCA_20663_211102_v001.fits', - 7 : 'SCA_22069_211228_v001.fits', - 8 : 'SCA_21641_211216_v001.fits', - 9 : 'SCA_21813_211219_v001.fits', - 10 : 'SCA_22078_211230_v001.fits', - 11 : 'SCA_21947_211226_v001.fits', - 12 : 'SCA_22077_211230_v001.fits', - 13 : 'SCA_22067_211227_v001.fits', - 14 : 'SCA_21814_211220_v001.fits', - 15 : 'SCA_21645_211228_v001.fits', - 16 : 'SCA_21643_211218_v001.fits', - 17 : 'SCA_21319_211211_v001.fits', - 18 : 'SCA_20833_211116_v001.fits', - } - -import os -import fitsio as fio -import galsim as galsim -import numpy as np -import galsim.roman as roman -from galsim.config import ParseValue -from roman_imsim.obseq import ObSeqDataLoader - -class get_pointing(object): - """ - Class to store stuff about the telescope - """ - def __init__(self, params, visit, SCA): - - self.params = params - file_name = params['input']['obseq_data']['file_name'] - obseq_data = ObSeqDataLoader(file_name, visit, SCA, logger=None) - self.filter = obseq_data.ob['filter'] - self.sca = obseq_data.ob['sca'] - self.visit = obseq_data.ob['visit'] - self.date = obseq_data.ob['date'] - self.exptime = obseq_data.ob['exptime'] - self.bpass = roman.getBandpasses()[self.filter] - self.WCS = roman.getWCS(world_pos = galsim.CelestialCoord(ra=obseq_data.ob['ra'], \ - dec=obseq_data.ob['dec']), - PA = obseq_data.ob['pa'], - date = self.date, - SCAs = self.sca, - PA_is_FPA = True - )[self.sca] - self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2),int(roman.n_pix/2))) - -class detector_effects(object): - """ - Class to simulate non-idealities and noise of roman detector images. - """ - def __init__(self, params, visit, sca, filter, logger, rng, rng_iter, sca_filepath=None): - self.sca = sca - self.filter = filter - if sca_filepath: - self.df = fio.FITS(os.path.join(sca_filepath, sca_number_to_file[self.sca])) - print('------- Using SCA files --------') - else: - self.df = None - print('------- Using simple detector model --------') - - self.params = params - self.rng = rng - self.noise = galsim.PoissonNoise(self.rng) - self.rng_np = np.random.default_rng(rng_iter) - self.pointing = get_pointing(self.params, visit, sca) - self.exptime = self.pointing.exptime - self.logger = logger - - self.force_cvz = False - if 'force_cvz' in self.params['image']['wcs']: - if self.params['image']['wcs']['force_cvz']: - self.force_cvz=True - - self.save_diff = False - if 'save_diff' in self.params['image']: - self.save_diff = bool(self.params['image']['save_diff']) - - class quantum_efficiency(object): - def __init__(self, params, logger, model='simple_model', sca_filepath=None): - self.model = getattr(self, model) - self.sca_filepath = sca_filepath - self.params = params - self.logger = logger - - def simple_model(self, image): - self.logger.info("Applying the simple QE model...") - return image - - def lab_model(self, image): - if self.sca_filepath is None: - self.logger.warning("No QE data file provided; a default value of QE = 1 will be used.") - return image - - self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.params['SCA']])) - self.logger.info("Applying the QE model derived from lab data…") - image.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array - return image - - def apply(self, image): - image = self.model(image) - return image - - class bfe(object): - def __init__(self, params, logger, model='simple_model', sca_filepath=None, saturation_level=100000): - self.model = getattr(self, model) - self.sca_filepath = sca_filepath - self.params = params - self.logger = logger - self.saturation_level = saturation_level - - def simple_model(sefl, image): - self.logger.info("No bfe effect will be applied.") - return image - - def lab_model(self, image): - """ - Apply brighter-fatter effect. - Brighter fatter effect is a non-linear effect that deflects photons due to the - the eletric field built by the accumulated charges. This effect exists in both - CCD and CMOS detectors and typically percent level change in charge. - The built-in electric field by the charges in pixels tends to repulse charges - to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. - This effect can also be understood effectly as change in pixel area and pixel - boundaries. - BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change - in the detector effect charaterization file. Kernel of the total pixel area, however, - is not sufficient. Image simulation of the brighter fatter effect requires the shift - of the four pixel boundaries. Before we get better data, we solve for the boundary - shift components from the kernel of total pixel area by assumming several symmetric constraints. - Input - im : Image - BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 - """ - if self.sca_filepath is None: - self.logger.warning("No BFE kernel data file provided; no bfe effect will be applied.") - return image - - self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.params['SCA']])) - - nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) - bin_size = 128 - n_max = 32 - m_max = 32 - num_grids = 4 - n_sub = n_max//num_grids - m_sub = m_max//num_grids - - ##======================================================================= - ## solve boundary shfit kernel aX components - ##======================================================================= - a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 - a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 - - ##solve aR aT aL aB for each a - for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) - for m in range(m_max): - a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) - - ## assume two parity symmetries - a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. - - r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection - B = (a[2,2], a[3,2], a[2,3], a[3,3], - a[4,2], a[2,4], a[3,4], a[4,4] ) - - A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], - [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], - [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], - [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], - [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], - [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], - [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], - [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) - - - s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] - - aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s3 , -s1 , s1 , s3 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) - - - aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], - [ -s7 , -s6 , -s4 , -s6 , -s7 ], - [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], - [ r*s6 , s5 , s2 , s5 , r*s6 ], - [ s7 , s6 , s4 , s6 , s7 ],]) - - - aL = aR[::-1, ::-1] - aB = aT[::-1, ::-1] - - - - - a_components[0, :,:, n, m] = aR[:,:] - a_components[1, :,:, n, m] = aT[:,:] - a_components[2, :,:, n, m] = aL[:,:] - a_components[3, :,:, n, m] = aB[:,:] - - ##============================= - ## Apply bfe to image - ##============================= - - ## pad and expand kernels - ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior - - # array_pad = self.saturate(image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 - array_pad = image.copy().array - saturation_array = np.ones_like(array_pad) * self.saturation_level - where_sat = np.where(array_pad > saturation_array) - array_pad[ where_sat ] = saturation_array[ where_sat ] - array_pad = array_pad[4:-4,4:-4] - array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array - - - dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] - - - ### run in sub grids to reduce memory - - ## pad and expand kernels - t = np.zeros((bin_size*n_sub, n_sub)) - for row in range(t.shape[0]): - t[row, row//(bin_size) ] =1 - - - - for gj in range(num_grids): - for gi in range(num_grids): - - a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) - - - for comp in range(4): - for j in range(2*nbfe+1): - for i in range(2*nbfe+1): - tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid - a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') - - #convolve aX_ij with Q_ij - for comp in range(4): - for dy in range(-nbfe, nbfe+1): - for dx in range(-nbfe, nbfe+1): - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ - *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] - - dj = int(np.sin(comp*np.pi/2)) - di = int(np.cos(comp*np.pi/2)) - - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ - array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) - - image.array[:,:] -= dQ_components.sum(axis=0) - image.array[:,1:] += dQ_components[0][:,:-1] - image.array[1:,:] += dQ_components[1][:-1,:] - image.array[:,:-1] += dQ_components[2][:,1:] - image.array[:-1,:] += dQ_components[3][1:,:] - - return image - - def apply(self, image): - image = self.model(image) - return image - - def set_diff(self, im=None): - if self.save_diff: - self.pre = im.copy() - self.pre.write('bg.fits', dir=self.params['diff_dir']) - return - - def diff(self, msg, im=None, verbose=True): - if self.save_diff: - diff = im-self.pre - diff.write('%s_diff.fits'%msg , dir=self.params['diff_dir']) - self.pre = im.copy() - im.write('%s_cumul.fits'%msg, dir=self.params['diff_dir']) - return - - def qe(self, im): - """ - Apply the wavelength-independent relative QE to the image. - Input - im : Image - RELQE1[4096,4096] : relative QE map - """ - - im.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array - return im - - # def bfe(self, im): - # """ - # Apply brighter-fatter effect. - # Brighter fatter effect is a non-linear effect that deflects photons due to the - # the eletric field built by the accumulated charges. This effect exists in both - # CCD and CMOS detectors and typically percent level change in charge. - # The built-in electric field by the charges in pixels tends to repulse charges - # to nearby pixels. Thus, the profile of more illuminous ojbect becomes broader. - # This effect can also be understood effectly as change in pixel area and pixel - # boundaries. - # BFE is defined in terms of the Antilogus coefficient kernel of total pixel area change - # in the detector effect charaterization file. Kernel of the total pixel area, however, - # is not sufficient. Image simulation of the brighter fatter effect requires the shift - # of the four pixel boundaries. Before we get better data, we solve for the boundary - # shift components from the kernel of total pixel area by assumming several symmetric constraints. - # Input - # im : Image - # BFE[nbfe+Delta y, nbfe+Delta x, y, x] : bfe coefficient kernel, nbfe=2 - # """ - - # nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) - # bin_size = 128 - # n_max = 32 - # m_max = 32 - # num_grids = 4 - # n_sub = n_max//num_grids - # m_sub = m_max//num_grids - - # ##======================================================================= - # ## solve boundary shfit kernel aX components - # ##======================================================================= - # a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 - # a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 - - # ##solve aR aT aL aB for each a - # for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) - # for m in range(m_max): - # a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) - - # ## assume two parity symmetries - # a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. - - # r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection - # B = (a[2,2], a[3,2], a[2,3], a[3,3], - # a[4,2], a[2,4], a[3,4], a[4,4] ) - - # A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], - # [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], - # [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], - # [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], - # [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], - # [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], - # [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], - # [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) - - - # s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] - - # aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], - # [ 0. , -s6 , -s5 , s5 , s6 ], - # [ 0. , -s3 , -s1 , s1 , s3 ], - # [ 0. , -s6 , -s5 , s5 , s6 ], - # [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) - - - # aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], - # [ -s7 , -s6 , -s4 , -s6 , -s7 ], - # [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], - # [ r*s6 , s5 , s2 , s5 , r*s6 ], - # [ s7 , s6 , s4 , s6 , s7 ],]) - - - # aL = aR[::-1, ::-1] - # aB = aT[::-1, ::-1] - - - - - # a_components[0, :,:, n, m] = aR[:,:] - # a_components[1, :,:, n, m] = aT[:,:] - # a_components[2, :,:, n, m] = aL[:,:] - # a_components[3, :,:, n, m] = aB[:,:] - - # ##============================= - # ## Apply bfe to image - # ##============================= - - # ## pad and expand kernels - # ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior - - # array_pad = self.saturate(im.copy()).array[4:-4,4:-4] # img of interest 4088x4088 - # array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array - - - # dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] - - - # ### run in sub grids to reduce memory - - # ## pad and expand kernels - # t = np.zeros((bin_size*n_sub, n_sub)) - # for row in range(t.shape[0]): - # t[row, row//(bin_size) ] =1 - - - - # for gj in range(num_grids): - # for gi in range(num_grids): - - # a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) - - - # for comp in range(4): - # for j in range(2*nbfe+1): - # for i in range(2*nbfe+1): - # tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid - # a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') - - # #convolve aX_ij with Q_ij - # for comp in range(4): - # for dy in range(-nbfe, nbfe+1): - # for dx in range(-nbfe, nbfe+1): - # dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - # += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ - # *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] - - # dj = int(np.sin(comp*np.pi/2)) - # di = int(np.cos(comp*np.pi/2)) - - # dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - # *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ - # array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) - - # im.array[:,:] -= dQ_components.sum(axis=0) - # im.array[:,1:] += dQ_components[0][:,:-1] - # im.array[1:,:] += dQ_components[1][:-1,:] - # im.array[:,:-1] += dQ_components[2][:,1:] - # im.array[:-1,:] += dQ_components[3][1:,:] - - - # return im - - def get_eff_sky_bg(self, pointing=None): - """ - Calculate effective sky background per pixel for nominal roman pixel scale. - - Input - pointing : Pointing object - radec : World coordinate position of image - """ - if pointing is None: - pointing = self.pointing - sky_level = roman.getSkyLevel(pointing.bpass, world_pos=pointing.radec, date=pointing.date) - sky_level *= (1.0 + roman.stray_light_fraction) * roman.pixel_scale**2 - - return sky_level - - def translate_cvz(self, orig_radec, field_ra=9.5, field_dec=-44, cvz_ra=61.24, cvz_dec=-48.42): - - ra = orig_radec.ra/galsim.degrees-field_ra - dec = orig_radec.dec/galsim.degrees-field_dec - ra += cvz_ra / np.cos(cvz_dec*np.pi/180) - dec += cvz_dec - return galsim.CelestialCoord(ra*galsim.degrees,dec*galsim.degrees) - - def setup_sky(self, im, pointing=None, force_cvz=False, stray_light=False, thermal_background=False): - """ - Setup sky - - First we get the amount of zodaical light for a position corresponding to the position of - the object. The results are provided in units of e-/arcsec^2, using the default Roman - exposure time since we did not explicitly specify one. Then we multiply this by a factor - >1 to account for the amount of stray light that is expected. If we do not provide a date - for the observation, then it will assume that it's the vernal equinox (sun at (0,0) in - ecliptic coordinates) in 2025. - - Input - im : Image - pointing : Pointing object - radec : World coordinate position of image - local_wcs : Local WCS - """ - if pointing is None: - pointing = self.pointing - if self.df is None: - self.dark_current_ = roman.dark_current * pointing.exptime - else: - self.dark_current_ = roman.dark_current * pointing.exptime + self.df['DARK'][:, :].flatten() * pointing.exptime - if self.df is None: - self.gain = roman.gain - else: - self.gain = self.df['GAIN'][:,:] - self.read_noise = galsim.GaussianNoise(self.rng, sigma=roman.read_noise) - - # Build current specification sky level if sky level not given - if force_cvz: - radec = self.translate_cvz(pointing.radec) - else: - radec = pointing.radec - - sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) - self.logger.debug('Adding sky_level = %s',sky_level) - if stray_light: - self.logger.debug('Stray light fraction = %s',roman.stray_light_fraction) - sky_level *= (1.0 + roman.stray_light_fraction) - - # Make a image of the sky that takes into account the spatially variable pixel scale. Note - # that makeSkyImage() takes a bit of time. If you do not care about the variable pixel - # scale, you could simply compute an approximate sky level in e-/pix by multiplying - # sky_level by roman.pixel_scale**2, and add that to final_image. - - # Create sky image - self.sky = galsim.Image(bounds=im.bounds, wcs=pointing.WCS) - pointing.WCS.makeSkyImage(self.sky, sky_level) - - # This image is in units of e-/pix. Finally we add the expected thermal backgrounds in this - # band. These are provided in e-/pix/s, so we have to multiply by the exposure time. - if thermal_background: - tb = roman.thermal_backgrounds[pointing.filter] * pointing.exptime - self.logger.debug('Adding thermal background: %s',tb) - self.sky += tb - - # Median of dark current is used here instead of mean since hot pixels contribute significantly to the mean. - # Stastistics of dark current for the current test detector file: (mean, std, median, max) ~ (35, 3050, 0.008, 1.2E6) (e-/p) - # Hot pixels could be removed in further analysis using the dq array. - self.sky_mean = np.mean(np.round((np.round(self.sky.array) + round(np.median(self.dark_current_)))/ np.mean(self.gain))) - - self.sky.addNoise(self.noise) - - def add_background(self, im, save_diff=False, draw_method='phot'): - """ - Add backgrounds to image (sky, thermal). - - First we get the amount of zodaical light for a position corresponding to the position of - the object. The results are provided in units of e-/arcsec^2, using the default Roman - exposure time since we did not explicitly specify one. Then we multiply this by a factor - >1 to account for the amount of stray light that is expected. If we do not provide a date - for the observation, then it will assume that it's the vernal equinox (sun at (0,0) in - ecliptic coordinates) in 2025. - - Input - im : Image - """ - - # If requested, dump an initial fits image to disk for diagnostics - if save_diff: - orig = im.copy() - orig.write('orig.fits') - - if draw_method != 'phot': - im.addNoise(self.noise) - - # Adding sky level to the image. - im += self.sky[self.sky.bounds & im.bounds] - - # If requested, dump a post-change fits image to disk for diagnostics - if save_diff: - prev = im.copy() - diff = prev-orig - diff.write('sky_a.fits') - - return im - - def recip_failure(self, im, exptime=None, alpha=roman.reciprocity_alpha, base_flux=1.0): - """ - Introduce reciprocity failure to image. - - Reciprocity, in the context of photography, is the inverse relationship between the - incident flux (I) of a source object and the exposure time (t) required to produce a given - response(p) in the detector, i.e., p = I*t. However, in NIR detectors, this relation does - not hold always. The pixel response to a high flux is larger than its response to a low - flux. This flux-dependent non-linearity is known as 'reciprocity failure', and the - approximate amount of reciprocity failure for the Roman detectors is known, so we can - include this detector effect in our images. - - Input - im : image - exptime : Exposure time - alpha : Reciprocity alpha - base_flux : Base flux - """ - - if exptime is None: - exptime = self.pointing.exptime - - # Add reciprocity effect - im.addReciprocityFailure(exp_time=exptime, alpha=alpha, base_flux=base_flux) - - return im - - def dark_current(self, im, exptime=None): - """ - Adding dark current to the image. - - Even when the detector is unexposed to any radiation, the electron-hole pairs that - are generated within the depletion region due to finite temperature are swept by the - high electric field at the junction of the photodiode. This small reverse bias - leakage current is referred to as 'dark current'. It is specified by the average - number of electrons reaching the detectors per unit time and has an associated - Poisson noise since it is a random event. - - Input - im : image - """ - - if exptime is None: - exptime = self.pointing.exptime - - if self.df is None: - self.dark_current_ = roman.dark_current * exptime - else: - self.dark_current_ = roman.dark_current * exptime + self.df['DARK'][:, :].flatten() * exptime - - if self.df is None: - self.im_dark = im.copy() - dark_current_ = self.dark_current_ - dark_noise = galsim.DeviateNoise(galsim.PoissonDeviate(self.rng, dark_current_)) - im.addNoise(dark_noise) - self.im_dark = im - self.im_dark - - else: - - dark_current_ = self.dark_current_.clip(0) - - # opt for numpy random geneator instead for speed - self.im_dark = self.rng_np.poisson(dark_current_).reshape(im.array.shape).astype(im.dtype) - im.array[:,:] += self.im_dark - - # NOTE: Sky level and dark current might appear like a constant background that can be - # simply subtracted. However, these contribute to the shot noise and matter for the - # non-linear effects that follow. Hence, these must be included at this stage of the - # image generation process. We subtract these backgrounds in the end. - - return im - - def saturate(self, im, saturation=100000): - """ - Clip the saturation level - Input - im : image - SATURATE[4096,4096] : saturation map - """ - - if self.df is None: - saturation_array = np.ones_like(im.array) * saturation - else: - saturation_array = self.df['SATURATE'][:,:] #4096x4096 array - where_sat = np.where(im.array > saturation_array) - im.array[ where_sat ] = saturation_array[ where_sat ] - - return im - - def deadpix(self, im): - """ - Apply dead pixel mask - Input - im : image - BADPIX[4096,4096] : bit mask with the first bit flags dead pixels - """ - - dead_mask = self.df['BADPIX'][:,:]&1 #4096x4096 array - im.array[ dead_mask>0 ]=0 - - return im - - def vtpe(self, im): - """ - Apply vertical trailing pixel effect. - The vertical trailing pixel effect (VTPE) is a non-linear effect that is - related to readout patterns. - Q'[j,i] = Q[j,i] + f( Q[j,i] - Q[j-1, i] ), - where f( dQ ) = dQ ( a + b * ln(1 + |dQ|/dQ0) ) - Input - im : image - VTPE[0,512,512] : coefficient a binned in 8x8 - VTPE[1,512,512] : coefficient a - VTPE[2,512,512] : coefficient dQ0 - """ - - # expand 512x512 arrays to 4096x4096 - - t = np.zeros((4096, 512)) - for row in range(t.shape[0]): - t[row, row//8] =1 - a_vtpe = t.dot(self.df['VTPE'][0,:,:][0]).dot(t.T) - ## NaN check - if np.isnan(a_vtpe).any(): - print("vtpe skipped due to NaN in file") - return im - b_vtpe = t.dot(self.df['VTPE'][1,:,:][0]).dot(t.T) - dQ0 = t.dot(self.df['VTPE'][2,:,:][0]).dot(t.T) - - dQ = im.array - np.roll(im.array, 1, axis=0) - dQ[0,:] *= 0 - - im.array[:,:] += dQ * ( a_vtpe + b_vtpe * np.log( 1. + np.abs(dQ)/dQ0 )) - return im - - def add_persistence(self, im, pointing=None): - """ - Applying the persistence effect. - - Even after reset, some charges from prior illuminations are trapped in defects of semiconductors. - Trapped charges are gradually released and generate the flux-dependent persistence signal. - Here we adopt the same fermi-linear model to describe the illumination dependence and time dependence - of the persistence effect for all SCAs. - - Input - im : image - pointing : pointing object - """ - if pointing is None: - pointing = self.pointing - - # load the dithers of sky images that were simulated - # dither_sca_array=np.loadtxt(self.params['image']['dither_from_file']).astype(int) - - # select adjacent exposures for the same sca (within 10*roman.exptime) - # dither_list_selected = dither_sca_array[dither_sca_array[:,1] == pointing.sca, 0] - # dither_list_selected = dither_list_selected[ np.abs(dither_list_selected - pointing.visit) < 10 ] - # p_list = np.array([get_pointing(self.params, i, self.sca) for i in dither_list_selected]) - - p_list = np.array([get_pointing(self.params, i, self.sca) for i in range(pointing.visit - 10, pointing.visit)]) - - dt_list = np.array([(pointing.date - p.date).total_seconds() for p in p_list]) - p_pers = p_list[ np.where((dt_list > 0) & (dt_list < pointing.exptime*10))] - - if self.df is None: - #iterate over previous exposures - for p in p_pers: - dt = (pointing.date - p.date).total_seconds() - pointing.exptime/2 ##avg time since end of exposures - self.params['output']['file_name']['items'] = [p.filter, p.visit, p.sca] - imfilename = ParseValue(self.params['output'], 'file_name', self.params, str)[0] - fn = os.path.join(self.params['output']['dir'], imfilename) - - # [TODO] - if not os.path.exists(fn): - continue - - ## apply all the effects that occured before persistence on the previouse exposures - ## since max of the sky background is of order 100, it is thus negligible for persistence - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4088, ymax=4088) - x = galsim.Image(bound_pad) - x.array[:,:] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] - x = self.recip_failure(x) - - # x = x.clip(0) ##remove negative stimulus - x.array.clip(0) ##remove negative stimulus - - im.array[:,:] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * pointing.exptime - - else: - - #setup parameters for persistence - Q01 = self.df['PERSIST'].read_header()['Q01'] - Q02 = self.df['PERSIST'].read_header()['Q02'] - Q03 = self.df['PERSIST'].read_header()['Q03'] - Q04 = self.df['PERSIST'].read_header()['Q04'] - Q05 = self.df['PERSIST'].read_header()['Q05'] - Q06 = self.df['PERSIST'].read_header()['Q06'] - alpha = self.df['PERSIST'].read_header()['ALPHA'] - - #iterate over previous exposures - for p in p_pers: - dt = (pointing.date - p.date).total_seconds() - pointing.exptime/2 ##avg time since end of exposures - fac_dt = (pointing.exptime/2.) / dt ##linear time dependence (approximate until we get t1 and Delat t of the data) - self.params['output']['file_name']['items'] = [p.filter, p.visit, p.sca] - imfilename = ParseValue(self.params['output'], 'file_name', self.params, str)[0] - fn = os.path.join(self.params['output']['dir'], imfilename) - - # [TODO] - if not os.path.exists(fn): - continue - - ## apply all the effects that occured before persistence on the previouse exposures - ## since max of the sky background is of order 100, it is thus negligible for persistence - ## same for brighter fatter effect - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) - x = galsim.Image(bound_pad) - x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] - x = self.qe(x).array[:,:] - - x = x.clip(0.1) ##remove negative and zero stimulus - - ## Do linear interpolation - a = np.zeros(x.shape) - a += ((x < Q01)) * x/Q01 - a += ((x >= Q01) & (x < Q02)) * (Q02-x)/(Q02-Q01) - im.array[:,:] += a * self.df['PERSIST'][0,:,:][0] * fac_dt - - - a = np.zeros(x.shape) - a += ((x >= Q01) & (x < Q02)) * (x-Q01)/(Q02-Q01) - a += ((x >= Q02) & (x < Q03)) * (Q03-x)/(Q03-Q02) - im.array[:,:] += a * self.df['PERSIST'][1,:,:][0] * fac_dt - - a = np.zeros(x.shape) - a += ((x >= Q02) & (x < Q03)) * (x-Q02)/(Q03-Q02) - a += ((x >= Q03) & (x < Q04)) * (Q04-x)/(Q04-Q03) - im.array[:,:] += a * self.df['PERSIST'][2,:,:][0] * fac_dt - - a = np.zeros(x.shape) - a += ((x >= Q03) & (x < Q04)) * (x-Q03)/(Q04-Q03) - a += ((x >= Q04) & (x < Q05)) * (Q05-x)/(Q05-Q04) - im.array[:,:] += a * self.df['PERSIST'][3,:,:][0] * fac_dt - - a = np.zeros(x.shape) - a += ((x >= Q04) & (x < Q05)) * (x-Q04)/(Q05-Q04) - a += ((x >= Q05) & (x < Q06)) * (Q06-x)/(Q06-Q05) - im.array[:,:] += a * self.df['PERSIST'][4,:,:][0] * fac_dt - - a = np.zeros(x.shape) - a += ((x >= Q05) & (x < Q06)) * (x-Q05)/(Q06-Q05) - a += ((x >= Q06)) * (x/Q06)**alpha ##avoid fractional power of negative values - im.array[:,:] += a * self.df['PERSIST'][5,:,:][0] * fac_dt - - - return im - - def nonlinearity(self, im, NLfunc=roman.NLfunc): - """ - Applying a quadratic non-linearity. - - Note that users who wish to apply some other nonlinearity function (perhaps for other NIR - detectors, or for CCDs) can use the more general nonlinearity routine, which uses the - following syntax: - final_image.applyNonlinearity(NLfunc=NLfunc) - with NLfunc being a callable function that specifies how the output image pixel values - should relate to the input ones. - - Input - im : Image - NLfunc : Nonlinearity function - """ - - # Apply the Roman nonlinearity routine, which knows all about the nonlinearity expected in - # the Roman detectors. Alternately, use a user-provided function. - if self.df is None: - im.applyNonlinearity(NLfunc=NLfunc) - else: - im.array[:,:] -= self.df['CNL'][0,:,:][0] * im.array**2 +\ - self.df['CNL'][1,:,:][0] * im.array**3 +\ - self.df['CNL'][2,:,:][0] * im.array**4 - - return im - - def interpix_cap(self, im, kernel=roman.ipc_kernel): - """ - Including Interpixel capacitance - - The voltage read at a given pixel location is influenced by the charges present in the - neighboring pixel locations due to capacitive coupling of sense nodes. This interpixel - capacitance effect is modeled as a linear effect that is described as a convolution of a - 3x3 kernel with the image. The Roman IPC routine knows about the kernel already, so the - user does not have to supply it. - - Input - im : image - kernel : Interpixel capacitance kernel - """ - - # Apply interpixel capacitance - if self.df is None: - im.applyIPC(kernel, edge_treatment='extend', fill_value=None) - else: - # pad the array by one pixel at the four edges - num_grids = 4 ### num_grids <= 8 - grid_size = 4096//num_grids - - array_pad = im.array[4:-4,4:-4] #it's an array instead of img - array_pad = np.pad(array_pad, [(5,5),(5,5)], mode='symmetric') #4098x4098 array - - K = self.df['IPC'][:, :, :, :] ##3,3,512, 512 - - t = np.zeros((grid_size, 512)) - for row in range(t.shape[0]): - t[row, row//( grid_size//512) ] =1 - - array_out = np.zeros( (4096, 4096)) - ##split job in sub_grids to reduce memory - for gj in range(num_grids): - for gi in range(num_grids): - K_pad = np.zeros( (3,3, grid_size+2, grid_size+2) ) - - for j in range(3): - for i in range(3): - tmp = (t.dot(K[j,i,:,:])).dot(t.T) #grid_sizexgrid_size - K_pad[j,i,:,:] = np.pad(tmp, [(1,1),(1,1)], mode='symmetric') - - for dy in range(-1, 2): - for dx in range(-1,2): - - array_out[ gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ - +=K_pad[ 1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size ] \ - *array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] - - im.array[:,:] = array_out - - return im - - - def add_read_noise(self, im): - """ - Adding read noise - - Read noise is the noise due to the on-chip amplifier that converts the charge into an - analog voltage. We already applied the Poisson noise due to the sky level, so read noise - should just be added as Gaussian noise - - Input - im : image - """ - - # Create noise realisation and apply it to image - if self.df is None: - self.im_read = im.copy() - im.addNoise(self.read_noise) - self.im_read = im - self.im_read - # self.sky.addNoise(self.read_noise) - else: - # use numpy random generator to draw 2-d noise map - read_noise = self.df['READ'][2,:,:].flatten() #flattened 4096x4096 array - self.im_read = self.rng_np.normal(loc=0., scale=read_noise).reshape(im.array.shape).astype(im.dtype) - im.array[:,:] += self.im_read - - # noise_array = self.rng_np.normal(loc=0., scale=read_noise) - # 4088x4088 img - # self.sky.array[:,:] += noise_array.reshape(im.array.shape)[4:-4, 4:-4].astype(self.sky.dtype) - - return im - - def e_to_ADU(self, im): - """ - We divide by the gain to convert from e- to ADU. Currently, the gain value in the Roman - module is just set to 1, since we don't know what the exact gain will be, although it is - expected to be approximately 1. Eventually, this may change when the camera is assembled, - and there may be a different value for each SCA. For now, there is just a single number, - which is equal to 1. - - Input - im : image - """ - if self.df is None: - return im / roman.gain - else: - bias = self.df['BIAS'][:,:] #4096x4096 img - t = np.zeros((4096, 32)) - for row in range(t.shape[0]): - t[row, row//128] =1 - gain_expand = (t.dot(self.gain)).dot(t.T) #4096x4096 gain img - im.array[:,:] = im.array/gain_expand + bias - return im - - def add_gain(self, im): - """ - We divide by the gain to convert from e- to ADU. - Input - im : image - GAIN : 32x32 float img in unit of e-/adu, mean(GAIN)~ 1.6 - """ - - gain = self.df['GAIN'][:, :] #32x32 img - - t = np.zeros((4096, 32)) - for row in range(t.shape[0]): - t[row, row//128] =1 - gain_expand = (t.dot(gain)).dot(t.T) #4096x4096 gain img - im.array[:,:] /= gain_expand - return im - - def add_bias(self, im): - """ - Add the voltage bias. - Input - im : image - BIAS : 4096x4096 uint16 bias img (in unit of DN), mean(bias) ~ 6.7k - """ - - bias = self.df['BIAS'][:,:] #4096x4096 img - - im.array[:,:] += bias - return im - - # def finalize_sky_im(self,im, pointing=None): - # """ - # Finalize sky background for subtraction from final image. Add dark current, - # convert to analog voltage, and quantize. - - # Input - # im : sky image - # """ - - # if pointing is None: - # pointing = self.pointing - - # if self.df is None: - # im.quantize() - # im += self.im_dark - # im = self.saturate(im) - # im += self.im_read - # im = self.e_to_ADU(im) - # im.quantize() - # else: - - # bound_pad = galsim.BoundsI( xmin=1, ymin=1, - # xmax=4096, ymax=4096) - # im_pad = galsim.Image(bound_pad) - # im_pad.array[4:-4, 4:-4] = im.array[:,:] - - # im_pad = self.qe(im_pad) - # im_pad = self.bfe(im_pad) - # im_pad = self.add_persistence(im_pad, pointing) - # im_pad.quantize() - # im_pad += self.im_dark - # im_pad = self.saturate(im_pad) - # im_pad = self.nonlinearity(im_pad) - # im_pad = self.interpix_cap(im_pad) - # im_pad = self.deadpix(im_pad) - # im_pad = self.vtpe(im_pad) - # im_pad += self.im_read - # im_pad = self.add_gain(im_pad) - # im_pad = self.add_bias(im_pad) - # im_pad.quantize() - # # output 4088x4088 img in uint16 - # im.array[:,:] = im_pad.array[4:-4, 4:-4] - # im = galsim.Image(im, dtype=np.uint16) - - # return im \ No newline at end of file diff --git a/roman_imsim/effects/background.py b/roman_imsim/effects/background.py index fdf16f03..260f1b1e 100644 --- a/roman_imsim/effects/background.py +++ b/roman_imsim/effects/background.py @@ -1,26 +1,26 @@ import os -import numpy as np -import fitsio as fio import galsim from roman_imsim.effects import roman_effects import galsim.roman as roman -from .utils import sca_number_to_file + class background(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.thermal_background = self.params['thermal_background'] if 'thermal_background' in self.params else False + self.thermal_background = self.params['thermal_background'] if 'thermal_background' \ + in self.params else False self.stray_light = self.params['stray_light'] if 'stray_light' in self.params else False - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def simple_model(self, image): if self.save_diff: orig = image.copy() - + self.logger.warning("Simple model will be applied for background.") pointing = self.pointing # Build current specification sky level if sky level not given @@ -29,28 +29,29 @@ def simple_model(self, image): else: radec = pointing.radec sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) - self.logger.debug('Adding sky_level = %s',sky_level) - + self.logger.debug('Adding sky_level = %s', sky_level) + if self.stray_light: - self.logger.debug('Stray light fraction = %s',roman.stray_light_fraction) + self.logger.debug('Stray light fraction = %s', roman.stray_light_fraction) sky_level *= (1.0 + roman.stray_light_fraction) # Create sky image self.sky = galsim.Image(bounds=image.bounds, wcs=pointing.WCS) pointing.WCS.makeSkyImage(self.sky, sky_level) if self.thermal_background: tb = roman.thermal_backgrounds[pointing.filter] * pointing.exptime - self.logger.debug('Adding thermal background: %s',tb) + self.logger.debug('Adding thermal background: %s', tb) self.sky += tb self.sky.addNoise(self.noise) - - # [TODO] Not entirely sure about this block, since the 'auto' option is meant to let the software choose which drawing method to use based on the total flux. + + # [TODO] Not entirely sure about this block, since the 'auto' option is meant to + # let the software choose which drawing method to use based on the total flux. if self.base['image']['draw_method'] not in ['phot', 'auto']: image.addNoise(self.noise) - + # Adding sky level to the image. image += self.sky[self.sky.bounds & image.bounds] if self.save_diff: prev = image.copy() diff = prev - orig diff.write(os.path.join(self.diff_dir, 'sky_a.fits')) - return image \ No newline at end of file + return image diff --git a/roman_imsim/effects/bias.py b/roman_imsim/effects/bias.py index 941ad24a..f4ce606f 100644 --- a/roman_imsim/effects/bias.py +++ b/roman_imsim/effects/bias.py @@ -1,34 +1,31 @@ import os -import numpy as np import fitsio as fio -import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class bias(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): self.logger.warning("No bias will be applied.") return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No bias data provided; no bias will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for bias.") - bias = self.df['BIAS'][:,:] #4096x4096 img + bias = self.df['BIAS'][:, :] # 4096x4096 img - image.array[:,:] += bias - return image \ No newline at end of file + image.array[:, :] += bias + return image diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/effects/brighter_fatter.py index 1a8d2285..cfa18b14 100644 --- a/roman_imsim/effects/brighter_fatter.py +++ b/roman_imsim/effects/brighter_fatter.py @@ -4,20 +4,21 @@ from roman_imsim.effects import roman_effects from .utils import sca_number_to_file + class brighter_fatter(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - # self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def simple_model(self, image): self.logger.warning("No bfe effect will be applied.") return image - + def lab_model(self, image): """ Apply brighter-fatter effect. @@ -41,10 +42,10 @@ def lab_model(self, image): self.logger.warning("No BFE kernel data file provided; no bfe effect will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for brighter-fatter effect.") - nbfe = 2 ## kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) + nbfe = 2 # kernel of bfe in shape (2 x nbfe+1)*(2 x nbfe+1) bin_size = 128 n_max = 32 m_max = 32 @@ -52,67 +53,62 @@ def lab_model(self, image): n_sub = n_max//num_grids m_sub = m_max//num_grids - ##======================================================================= - ## solve boundary shfit kernel aX components - ##======================================================================= - a_area = self.df['BFE'][:,:,:,:] #5x5x32x32 - a_components = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, n_max, m_max) ) #4x5x5x32x32 + # ======================================================================= + # solve boundary shfit kernel aX components + # ======================================================================= + a_area = self.df['BFE'][:, :, :, :] # 5x5x32x32 + a_components = np.zeros((4, 2*nbfe+1, 2*nbfe+1, n_max, m_max)) # 4x5x5x32x32 - ##solve aR aT aL aB for each a - for n in range(n_max): #m_max and n_max = 32 (binned in 128x128) + # solve aR aT aL aB for each a + for n in range(n_max): # m_max and n_max = 32 (binned in 128x128) for m in range(m_max): - a = a_area[:,:, n, m] ## a in (2 x nbfe+1)*(2 x nbfe+1) - - ## assume two parity symmetries - a = ( a + np.fliplr(a) + np.flipud(a) + np.flip(a) )/4. - - r = 0.5* ( 3.25/4.25 )**(1.5) / 1.5 ## source-boundary projection - B = (a[2,2], a[3,2], a[2,3], a[3,3], - a[4,2], a[2,4], a[3,4], a[4,4] ) - - A = np.array( [ [ -2 , -2 , 0 , 0 , 0 , 0 , 0 ], - [ 0 , 1 , 0 , -1 , -2 , 0 , 0 ], - [ 1 , 0 , -1 , 0 , -2 , 0 , 0 ], - [ 0 , 0 , 0 , 0 , 2 , -2 , 0 ], - [ 0 , 0 , 0 , 1 , 0 ,-2*r, 0 ], - [ 0 , 0 , 1 , 0 , 0 ,-2*r, 0 ], - [ 0 , 0 , 0 , 0 , 0 , 1+r, -1 ], - [ 0 , 0 , 0 , 0 , 0 , 0 , 2 ] ]) - - - s1,s2,s3,s4,s5,s6,s7 = np.linalg.lstsq(A, B, rcond=None)[0] - - aR = np.array( [[ 0. , -s7 ,-r*s6 , r*s6 , s7 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s3 , -s1 , s1 , s3 ], - [ 0. , -s6 , -s5 , s5 , s6 ], - [ 0. , -s7 ,-r*s6 , r*s6 , s7 ],]) - - - aT = np.array( [[ 0. , 0. , 0. , 0. , 0. ], - [ -s7 , -s6 , -s4 , -s6 , -s7 ], - [ -r*s6 , -s5 , -s2 , -s5 , -r*s6 ], - [ r*s6 , s5 , s2 , s5 , r*s6 ], - [ s7 , s6 , s4 , s6 , s7 ],]) - + a = a_area[:, :, n, m] # a in (2 x nbfe+1)*(2 x nbfe+1) + + # assume two parity symmetries + a = (a + np.fliplr(a) + np.flipud(a) + np.flip(a))/4. + + r = 0.5 * (3.25/4.25)**(1.5) / 1.5 # source-boundary projection + B = (a[2, 2], a[3, 2], a[2, 3], a[3, 3], + a[4, 2], a[2, 4], a[3, 4], a[4, 4]) + + A = np.array([[-2 , -2 , 0 , 0 , 0 , 0 , 0], + [0 , 1 , 0 , -1 , -2 , 0 , 0], + [1 , 0 , -1 , 0 , -2 , 0 , 0], + [0 , 0 , 0 , 0 , 2 , -2 , 0], + [0 , 0 , 0 , 1 , 0 , -2*r, 0], + [0 , 0 , 1 , 0 , 0 , -2*r, 0], + [0 , 0 , 0 , 0 , 0 , 1+r, -1], + [0 , 0 , 0 , 0 , 0 , 0 , 2]]) + + s1, s2, s3, s4, s5, s6, s7 = np.linalg.lstsq(A, B, rcond=None)[0] + + aR = np.array([[0. , -s7 , -r*s6 , r*s6 , s7], + [0. , -s6 , -s5 , s5 , s6], + [0. , -s3 , -s1 , s1 , s3], + [0. , -s6 , -s5 , s5 , s6], + [0. , -s7 , -r*s6 , r*s6 , s7],]) + + aT = np.array([[0. , 0. , 0. , 0. , 0.], + [-s7 , -s6 , -s4 , -s6 , -s7], + [-r*s6 , -s5 , -s2 , -s5 , -r*s6], + [r*s6 , s5 , s2 , s5 , r*s6], + [s7 , s6 , s4 , s6 , s7],]) aL = aR[::-1, ::-1] aB = aT[::-1, ::-1] + a_components[0, :, :, n, m] = aR[:, :] + a_components[1, :, :, n, m] = aT[:, :] + a_components[2, :, :, n, m] = aL[:, :] + a_components[3, :, :, n, m] = aB[:, :] + # ============================= + # Apply bfe to image + # ============================= - - a_components[0, :,:, n, m] = aR[:,:] - a_components[1, :,:, n, m] = aT[:,:] - a_components[2, :,:, n, m] = aL[:,:] - a_components[3, :,:, n, m] = aB[:,:] - - ##============================= - ## Apply bfe to image - ##============================= - - ## pad and expand kernels - ## The img is clipped by the saturation level here to cap the brighter fatter effect and avoid unphysical behavior + # pad and expand kernels + # The img is clipped by the saturation level here to cap the brighter fatter effect + # and avoid unphysical behavior # array_pad = image.copy().array # saturation_array = np.ones_like(array_pad) * self.saturation_level @@ -120,53 +116,62 @@ def lab_model(self, image): # array_pad[ where_sat ] = saturation_array[ where_sat ] # array_pad = array_pad[4:-4,4:-4] saturate = self.cross_refer('saturate') - array_pad = saturate.apply(image = image.copy()).array[4:-4,4:-4] # img of interest 4088x4088 - array_pad = np.pad(array_pad, [(4+nbfe,4+nbfe),(4+nbfe,4+nbfe)], mode='symmetric') #4100x4100 array - + array_pad = saturate.apply(image=image.copy()).array[4:-4, 4:-4] # img of interest 4088x4088 + array_pad = np.pad(array_pad, [(4+nbfe, 4+nbfe), (4+nbfe, 4+nbfe)], + mode='symmetric') # 4100x4100 array - dQ_components = np.zeros( (4, bin_size*n_max, bin_size*m_max) ) #(4, 4096, 4096) in order of [aR, aT, aL, aB] + # (4, 4096, 4096) in order of [aR, aT, aL, aB] + dQ_components = np.zeros((4, bin_size*n_max, bin_size*m_max)) + # run in sub grids to reduce memory - ### run in sub grids to reduce memory - - ## pad and expand kernels + # pad and expand kernels t = np.zeros((bin_size*n_sub, n_sub)) for row in range(t.shape[0]): - t[row, row//(bin_size) ] =1 - - + t[row, row//(bin_size)] = 1 for gj in range(num_grids): for gi in range(num_grids): - a_components_pad = np.zeros( (4, 2*nbfe+1, 2*nbfe+1, bin_size*n_sub+2*nbfe, bin_size*m_sub+2*nbfe) ) #(4,5,5,sub_grid,sub_grid) - + # (4,5,5,sub_grid,sub_grid) + a_components_pad = np.zeros((4, 2*nbfe+1, 2*nbfe+1, bin_size + * n_sub+2*nbfe, bin_size*m_sub+2*nbfe)) for comp in range(4): for j in range(2*nbfe+1): for i in range(2*nbfe+1): - tmp = (t.dot( a_components[comp,j,i,gj*n_sub:(gj+1)*n_sub,gi*m_sub:(gi+1)*m_sub] ) ).dot(t.T) #sub_grid*sub_grid - a_components_pad[comp, j, i, :, :] = np.pad(tmp, [(nbfe,nbfe),(nbfe,nbfe)], mode='symmetric') + # sub_grid*sub_grid + tmp = (t.dot(a_components[comp, j, i, gj*n_sub:(gj+1) + * n_sub, gi*m_sub:(gi+1)*m_sub])).dot(t.T) + a_components_pad[comp, j, i, :, :] = np.pad( + tmp, [(nbfe, nbfe), (nbfe, nbfe)], mode='symmetric') - #convolve aX_ij with Q_ij + # convolve aX_ij with Q_ij for comp in range(4): for dy in range(-nbfe, nbfe+1): for dx in range(-nbfe, nbfe+1): - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, nbfe-dx:nbfe-dx+bin_size*m_sub ]\ - *array_pad[ -dy + nbfe + gj*bin_size*n_sub : -dy + nbfe+ (gj+1)*bin_size*n_sub , -dx + nbfe + gi*bin_size*m_sub : -dx + nbfe + (gi+1)*bin_size*m_sub ] + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub, + gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, + nbfe-dx:nbfe-dx+bin_size*m_sub]\ + * array_pad[-dy + nbfe + gj*bin_size*n_sub : -dy + nbfe + + (gj+1)*bin_size*n_sub, -dx + nbfe + gi*bin_size*m_sub : -dx + + nbfe + (gi+1)*bin_size*m_sub] dj = int(np.sin(comp*np.pi/2)) di = int(np.cos(comp*np.pi/2)) - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - *= 0.5*(array_pad[ nbfe + gj*bin_size*n_sub : nbfe+ (gj+1)*bin_size*n_sub , nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub ] +\ - array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe+ (gj+1)*bin_size*n_sub , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub] ) + dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , + gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ + *= 0.5*(array_pad[nbfe + gj*bin_size*n_sub : nbfe + (gj+1)*bin_size*n_sub, + nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub] + + array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe + (gj+1)*bin_size*n_sub + , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub]) - image.array[:,:] -= dQ_components.sum(axis=0) - image.array[:,1:] += dQ_components[0][:,:-1] - image.array[1:,:] += dQ_components[1][:-1,:] - image.array[:,:-1] += dQ_components[2][:,1:] - image.array[:-1,:] += dQ_components[3][1:,:] + image.array[:, :] -= dQ_components.sum(axis=0) + image.array[:, 1:] += dQ_components[0][:, :-1] + image.array[1:, :] += dQ_components[1][:-1, :] + image.array[:, :-1] += dQ_components[2][:, 1:] + image.array[:-1, :] += dQ_components[3][1:, :] - return image \ No newline at end of file + return image diff --git a/roman_imsim/effects/dark_current.py b/roman_imsim/effects/dark_current.py index d8df88e5..bdcc417d 100644 --- a/roman_imsim/effects/dark_current.py +++ b/roman_imsim/effects/dark_current.py @@ -1,21 +1,21 @@ import os -import numpy as np import fitsio as fio import galsim import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class dark_current(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def simple_model(self, image): self.logger.warning("Simple model will be applied for dark current.") exptime = self.pointing.exptime @@ -26,18 +26,18 @@ def simple_model(self, image): image.addNoise(dark_noise) self.im_dark = image - self.im_dark return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No dark current file provided; no dark current will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + exptime = self.pointing.exptime self.logger.warning("Lab measured model will be applied for dark current.") self.dark_current_ = roman.dark_current * exptime + self.df['DARK'][:, :].flatten() * exptime dark_current_ = self.dark_current_.clip(0) # opt for numpy random geneator instead for speed self.im_dark = self.rng_np.poisson(dark_current_).reshape(image.array.shape).astype(image.dtype) - image.array[:,:] += self.im_dark - return image \ No newline at end of file + image.array[:, :] += self.im_dark + return image diff --git a/roman_imsim/effects/dead_pix.py b/roman_imsim/effects/dead_pix.py index 88eddeaf..69332310 100644 --- a/roman_imsim/effects/dead_pix.py +++ b/roman_imsim/effects/dead_pix.py @@ -1,34 +1,31 @@ import os -import numpy as np import fitsio as fio -import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class dead_pix(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): self.logger.warning("No dead pixel will be applied.") return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No bad pixel data file provided; no dead pixel will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for dead pixel.") - dead_mask = self.df['BADPIX'][:,:]&1 #4096x4096 array - image.array[ dead_mask>0 ]=0 - - return image \ No newline at end of file + dead_mask = self.df['BADPIX'][:, :] & 1 # 4096x4096 array + image.array[dead_mask > 0] = 0 + + return image diff --git a/roman_imsim/effects/gain.py b/roman_imsim/effects/gain.py index 2dbe0884..650210e8 100644 --- a/roman_imsim/effects/gain.py +++ b/roman_imsim/effects/gain.py @@ -1,40 +1,39 @@ import os import numpy as np import fitsio as fio -import galsim import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class gain(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): self.logger.warning("Simple model will be applied for gain.") image /= roman.gain return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No gain data provided; galsim.roman.gain will be applied.") image /= roman.gain return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for gain.") - gain_map = self.df['GAIN'][:, :] #32x32 img + gain_map = self.df['GAIN'][:, :] # 32x32 img t = np.zeros((4096, 32)) for row in range(t.shape[0]): - t[row, row//128] =1 - gain_expand = (t.dot(gain_map)).dot(t.T) #4096x4096 gain img - image.array[:,:] /= gain_expand - return image \ No newline at end of file + t[row, row//128] = 1 + gain_expand = (t.dot(gain_map)).dot(t.T) # 4096x4096 gain img + image.array[:, :] /= gain_expand + return image diff --git a/roman_imsim/effects/interpix_cap.py b/roman_imsim/effects/interpix_cap.py index 2fc91eb9..aba6125e 100644 --- a/roman_imsim/effects/interpix_cap.py +++ b/roman_imsim/effects/interpix_cap.py @@ -1,65 +1,65 @@ import os import numpy as np import fitsio as fio -import galsim import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class interpix_cap(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): self.logger.warning("Simple model will be applied for IPC effect.") - kernel=roman.ipc_kernel + kernel = roman.ipc_kernel image.applyIPC(kernel, edge_treatment='extend', fill_value=None) return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No IPC kernel data file provided; no IPC effect will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for IPC effect.") # pad the array by one pixel at the four edges - num_grids = 4 ### num_grids <= 8 + num_grids = 4 # num_grids <= 8 grid_size = 4096//num_grids - array_pad = image.array[4:-4,4:-4] #it's an array instead of img - array_pad = np.pad(array_pad, [(5,5),(5,5)], mode='symmetric') #4098x4098 array - - K = self.df['IPC'][:, :, :, :] ##3,3,512, 512 + array_pad = image.array[4:-4, 4:-4] # it's an array instead of img + array_pad = np.pad(array_pad, [(5, 5), (5, 5)], mode='symmetric') # 4098x4098 array + + K = self.df['IPC'][:, :, :, :] # 3,3,512, 512 t = np.zeros((grid_size, 512)) for row in range(t.shape[0]): - t[row, row//( grid_size//512) ] =1 + t[row, row//(grid_size//512)] = 1 - array_out = np.zeros( (4096, 4096)) - ##split job in sub_grids to reduce memory + array_out = np.zeros((4096, 4096)) + # split job in sub_grids to reduce memory for gj in range(num_grids): for gi in range(num_grids): - K_pad = np.zeros( (3,3, grid_size+2, grid_size+2) ) + K_pad = np.zeros((3, 3, grid_size+2, grid_size+2)) for j in range(3): for i in range(3): - tmp = (t.dot(K[j,i,:,:])).dot(t.T) #grid_sizexgrid_size - K_pad[j,i,:,:] = np.pad(tmp, [(1,1),(1,1)], mode='symmetric') + tmp = (t.dot(K[j, i, :, :])).dot(t.T) # grid_sizexgrid_size + K_pad[j, i, :, :] = np.pad(tmp, [(1, 1), (1, 1)], mode='symmetric') for dy in range(-1, 2): - for dx in range(-1,2): + for dx in range(-1, 2): - array_out[ gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ - +=K_pad[ 1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size ] \ - *array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] + array_out[gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ + += K_pad[1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size] \ + * array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, + 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] - image.array[:,:] = array_out - return image \ No newline at end of file + image.array[:, :] = array_out + return image diff --git a/roman_imsim/effects/nonlinearity.py b/roman_imsim/effects/nonlinearity.py index dfa9359f..ef1cc96d 100644 --- a/roman_imsim/effects/nonlinearity.py +++ b/roman_imsim/effects/nonlinearity.py @@ -4,6 +4,7 @@ from roman_imsim.effects import roman_effects from .utils import sca_number_to_file + class nonlinearity(roman_effects): """ Applying a quadratic non-linearity. @@ -15,31 +16,33 @@ class nonlinearity(roman_effects): with NLfunc being a callable function that specifies how the output image pixel values should relate to the input ones. """ - + def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def simple_model(self, image): self.logger.info("Galsim.roman.NLfunc will be applied for simulating non-linearity effect.") image.applyNonlinearity(NLfunc=roman.NLfunc) return image - + def lab_model(self, image): if self.sca_filepath is None: - self.logger.warning("No non-linearity data file provided; no non-linearity effect will be applied.") + self.logger.warning( + "No non-linearity data file provided; no non-linearity effect will be applied.") return image - + self.logger.warning("Lab measured model will be applied for non-linearity effect.") self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - - image.array[:,:] -= self.df['CNL'][0,:,:][0] * image.array**2 +\ - self.df['CNL'][1,:,:][0] * image.array**3 +\ - self.df['CNL'][2,:,:][0] * image.array**4 - - return image \ No newline at end of file + + image.array[:, :] -= self.df['CNL'][0, :, :][0] * image.array**2 +\ + self.df['CNL'][1, :, :][0] * image.array**3 +\ + self.df['CNL'][2, :, :][0] * image.array**4 + + return image diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/effects/persistence.py index f4620e5c..330f3ad7 100644 --- a/roman_imsim/effects/persistence.py +++ b/roman_imsim/effects/persistence.py @@ -3,26 +3,28 @@ import fitsio as fio import galsim from roman_imsim.effects import roman_effects -from galsim.config import ParseValue from .utils import sca_number_to_file, get_pointing + class persistence(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + p_list = np.array([get_pointing(self.base, i, self.sca) for i in range(self.visit - 10, self.visit)]) dt_list = np.array([(self.pointing.date - p.date).total_seconds() for p in p_list]) - self.p_pers = p_list[ np.where((dt_list > 0) & (dt_list < self.pointing.exptime*10))] - + self.p_pers = p_list[np.where((dt_list > 0) & (dt_list < self.pointing.exptime*10))] + def simple_model(self, image): self.logger.warning("Simple model will be applied for persistence effect.") for p in self.p_pers: - dt = (self.pointing.date - p.date).total_seconds() - self.pointing.exptime/2 ##avg time since end of exposures + dt = (self.pointing.date - p.date).total_seconds() - \ + self.pointing.exptime/2 # avg time since end of exposures # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) @@ -32,31 +34,32 @@ def simple_model(self, image): if not os.path.exists(fn): continue - ## apply all the effects that occured before persistence on the previouse exposures - ## since max of the sky background is of order 100, it is thus negligible for persistence - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) + # apply all the effects that occured before persistence on the previouse exposures + # since max of the sky background is of order 100, it is thus negligible for persistence + bound_pad = galsim.BoundsI(xmin=1, ymin=1, + xmax=4096, ymax=4096) x = galsim.Image(bound_pad) - x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] - + x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] + recip_failure = self.cross_refer('recip_failure') - x = recip_failure.apply(image = x) + x = recip_failure.apply(image=x) + + x.array.clip(0) # remove negative stimulus - x.array.clip(0) ##remove negative stimulus + image.array[:, + :] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * self.pointing.exptime - image.array[:,:] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * self.pointing.exptime - return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No persistence data file provided; no persistence effect will be applied.") return image - + self.logger.warning("Lab measured model will be applied for persistence effect.") self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - - #setup parameters for persistence + + # setup parameters for persistence Q01 = self.df['PERSIST'].read_header()['Q01'] Q02 = self.df['PERSIST'].read_header()['Q02'] Q03 = self.df['PERSIST'].read_header()['Q03'] @@ -65,13 +68,15 @@ def lab_model(self, image): Q06 = self.df['PERSIST'].read_header()['Q06'] alpha = self.df['PERSIST'].read_header()['ALPHA'] - #iterate over previous exposures + # iterate over previous exposures for p in self.p_pers: - dt = (self.pointing.date - p.date).total_seconds() - self.pointing.exptime/2 ##avg time since end of exposures - fac_dt = (self.pointing.exptime/2.) / dt ##linear time dependence (approximate until we get t1 and Delat t of the data) + dt = (self.pointing.date - p.date).total_seconds() - \ + self.pointing.exptime/2 # avg time since end of exposures + # linear time dependence (approximate until we get t1 and Delat t of the data) + fac_dt = (self.pointing.exptime/2.) / dt # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] - + imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) fn = os.path.join(self.base['output']['dir'], imfilename) @@ -79,50 +84,49 @@ def lab_model(self, image): if not os.path.exists(fn): continue - ## apply all the effects that occured before persistence on the previouse exposures - ## since max of the sky background is of order 100, it is thus negligible for persistence - ## same for brighter fatter effect - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) + # apply all the effects that occured before persistence on the previouse exposures + # since max of the sky background is of order 100, it is thus negligible for persistence + # same for brighter fatter effect + bound_pad = galsim.BoundsI(xmin=1, ymin=1, + xmax=4096, ymax=4096) x = galsim.Image(bound_pad) - x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:,:] + x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] # x = self.qe(x).array[:,:] - + qe = self.cross_refer('quantum_efficiency') - x = qe.apply(image = x).array[:, :] + x = qe.apply(image=x).array[:, :] - x = x.clip(0.1) ##remove negative and zero stimulus + x = x.clip(0.1) # remove negative and zero stimulus - ## Do linear interpolation + # Do linear interpolation a = np.zeros(x.shape) a += ((x < Q01)) * x/Q01 a += ((x >= Q01) & (x < Q02)) * (Q02-x)/(Q02-Q01) - image.array[:,:] += a * self.df['PERSIST'][0,:,:][0] * fac_dt - + image.array[:, :] += a * self.df['PERSIST'][0, :, :][0] * fac_dt a = np.zeros(x.shape) a += ((x >= Q01) & (x < Q02)) * (x-Q01)/(Q02-Q01) a += ((x >= Q02) & (x < Q03)) * (Q03-x)/(Q03-Q02) - image.array[:,:] += a * self.df['PERSIST'][1,:,:][0] * fac_dt + image.array[:, :] += a * self.df['PERSIST'][1, :, :][0] * fac_dt a = np.zeros(x.shape) a += ((x >= Q02) & (x < Q03)) * (x-Q02)/(Q03-Q02) a += ((x >= Q03) & (x < Q04)) * (Q04-x)/(Q04-Q03) - image.array[:,:] += a * self.df['PERSIST'][2,:,:][0] * fac_dt + image.array[:, :] += a * self.df['PERSIST'][2, :, :][0] * fac_dt a = np.zeros(x.shape) a += ((x >= Q03) & (x < Q04)) * (x-Q03)/(Q04-Q03) a += ((x >= Q04) & (x < Q05)) * (Q05-x)/(Q05-Q04) - image.array[:,:] += a * self.df['PERSIST'][3,:,:][0] * fac_dt + image.array[:, :] += a * self.df['PERSIST'][3, :, :][0] * fac_dt a = np.zeros(x.shape) a += ((x >= Q04) & (x < Q05)) * (x-Q04)/(Q05-Q04) a += ((x >= Q05) & (x < Q06)) * (Q06-x)/(Q06-Q05) - image.array[:,:] += a * self.df['PERSIST'][4,:,:][0] * fac_dt + image.array[:, :] += a * self.df['PERSIST'][4, :, :][0] * fac_dt a = np.zeros(x.shape) a += ((x >= Q05) & (x < Q06)) * (x-Q05)/(Q06-Q05) - a += ((x >= Q06)) * (x/Q06)**alpha ##avoid fractional power of negative values - image.array[:,:] += a * self.df['PERSIST'][5,:,:][0] * fac_dt + a += ((x >= Q06)) * (x/Q06)**alpha # avoid fractional power of negative values + image.array[:, :] += a * self.df['PERSIST'][5, :, :][0] * fac_dt - return image \ No newline at end of file + return image diff --git a/roman_imsim/effects/quantum_efficiency.py b/roman_imsim/effects/quantum_efficiency.py index 589e7003..f175ef62 100644 --- a/roman_imsim/effects/quantum_efficiency.py +++ b/roman_imsim/effects/quantum_efficiency.py @@ -1,26 +1,25 @@ import os -import numpy as np import fitsio as fio -import galsim from roman_imsim.effects import roman_effects -import galsim.roman as roman from .utils import sca_number_to_file + class quantum_efficiency(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No QE data file provided; a default value of QE = 1 will be used.") return image - + self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for quantum efficiency.") - image.array[:,:] *= self.df['RELQE1'][:,:] #4096x4096 array - return image \ No newline at end of file + image.array[:, :] *= self.df['RELQE1'][:, :] # 4096x4096 array + return image diff --git a/roman_imsim/effects/read_noise.py b/roman_imsim/effects/read_noise.py index 797544aa..dd6080c2 100644 --- a/roman_imsim/effects/read_noise.py +++ b/roman_imsim/effects/read_noise.py @@ -1,20 +1,17 @@ import os -import numpy as np import fitsio as fio -import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class read_noise(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): @@ -24,15 +21,15 @@ def simple_model(self, image): self.im_read = image - self.im_read # self.sky.addNoise(self.read_noise) return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No read noise data file provided; no read noise will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for read noise.") - rdn = self.df['READ'][2,:,:].flatten() #flattened 4096x4096 array + rdn = self.df['READ'][2, :, :].flatten() # flattened 4096x4096 array self.im_read = self.rng_np.normal(loc=0., scale=rdn).reshape(image.array.shape).astype(image.dtype) - image.array[:,:] += self.im_read - return image \ No newline at end of file + image.array[:, :] += self.im_read + return image diff --git a/roman_imsim/effects/recip_failure.py b/roman_imsim/effects/recip_failure.py index aecffa53..e0bc442b 100644 --- a/roman_imsim/effects/recip_failure.py +++ b/roman_imsim/effects/recip_failure.py @@ -1,26 +1,23 @@ -import os -import numpy as np -import fitsio as fio -import galsim from roman_imsim.effects import roman_effects import galsim.roman as roman -from .utils import sca_number_to_file + class recip_failure(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.alpha = self.params['alpha'] if 'alpha' in self.params else roman.reciprocity_alpha self.base_flux = self.params['base_flux'] if 'base_flux' in self.params else 1.0 - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - + def simple_model(self, image): # Add reciprocity effect self.logger.warning("Simple model will be applied for reciprocity failure effect.") exptime = self.pointing.exptime image.addReciprocityFailure(exp_time=exptime, alpha=self.alpha, base_flux=self.base_flux) - return image \ No newline at end of file + return image diff --git a/roman_imsim/effects/roman_effects.py b/roman_imsim/effects/roman_effects.py index e86f01b7..cadab6d0 100644 --- a/roman_imsim/effects/roman_effects.py +++ b/roman_imsim/effects/roman_effects.py @@ -1,37 +1,35 @@ -import os -import fitsio as fio import galsim as galsim import numpy as np -import galsim.roman as roman -from galsim.config import ParseValue from .utils import get_pointing import roman_imsim.effects as effects - + + class roman_effects(object): """ Class to simulate non-idealities and noise of roman detector images. """ + def __init__(self, params, base, logger, rng, rng_iter=None): - self.params = params - self.base = base - self.visit = int(self.base['input']['obseq_data']['visit']) - self.sca = base['image']['SCA'] + self.params = params + self.base = base + self.visit = int(self.base['input']['obseq_data']['visit']) + self.sca = base['image']['SCA'] self.filter = base['image']['filter'] self.sca_filepath = base['image']['sca_filepath'] self.rng_iter = rng_iter if rng_iter else self.visit * self.sca - - self.rng = rng - self.noise = galsim.PoissonNoise(self.rng) - self.rng_np = np.random.default_rng(self.rng_iter) - self.pointing = get_pointing(self.base, self.visit, self.sca) - self.exptime = self.pointing.exptime - self.logger = logger - + + self.rng = rng + self.noise = galsim.PoissonNoise(self.rng) + self.rng_np = np.random.default_rng(self.rng_iter) + self.pointing = get_pointing(self.base, self.visit, self.sca) + self.exptime = self.pointing.exptime + self.logger = logger + self.force_cvz = False if 'force_cvz' in self.base['image']['wcs']: if self.base['image']['wcs']['force_cvz']: - self.force_cvz=True - + self.force_cvz = True + self.save_diff = False if 'save_diff' in self.base['image']: self.save_diff = bool(self.base['image']['save_diff']) @@ -39,15 +37,16 @@ def __init__(self, params, base, logger, rng, rng_iter=None): self.diff_dir = self.base['output']['diff_dir'] else: self.diff_dir = self.base['output']['dir'] - + def simple_model(self, image): self.logger.info("Applying the default model...") return image - + def cross_refer(self, effect_name): if effect_name not in self.base['image']['add_effects']: try: - effect = getattr(effects, effect_name)({'model':'simple_model'}, self.base, self.logger, self.rng) + effect = getattr(effects, effect_name)( + {'model': 'simple_model'}, self.base, self.logger, self.rng) except Exception as e: self.logger.warning(e) # self.logger.warning("Effect %s is not implemented!"%(effect_name)) @@ -61,11 +60,11 @@ def cross_refer(self, effect_name): # self.logger.warning("Effect %s is not implemented!"%(effect_name)) return None return effect - + def apply(self, image): - image = self.model(image) - return image - + image = self.model(image) + return image + def set_diff(self, im=None): if self.save_diff: self.pre = im.copy() @@ -78,4 +77,4 @@ def diff(self, msg, im=None, verbose=True): diff.write('%s_diff.fits'%msg , dir=self.diff_dir) self.pre = im.copy() im.write('%s_cumul.fits'%msg, dir=self.diff_dir) - return \ No newline at end of file + return diff --git a/roman_imsim/effects/saturate.py b/roman_imsim/effects/saturate.py index 47d82a3a..194d56df 100644 --- a/roman_imsim/effects/saturate.py +++ b/roman_imsim/effects/saturate.py @@ -1,39 +1,39 @@ import os import numpy as np import fitsio as fio -import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class saturate(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model - - self.saturation_level = self.params['saturation_level'] if 'saturation_level' in self.params else 100000 - + + self.saturation_level = self.params['saturation_level'] \ + if 'saturation_level' in self.params else 100000 + def simple_model(self, image): self.logger.warning("Simple model will be applied for saturation.") saturation_array = np.ones_like(image.array) * self.saturation_level where_sat = np.where(image.array > saturation_array) - image.array[ where_sat ] = saturation_array[ where_sat ] + image.array[where_sat] = saturation_array[where_sat] return image - + def lab_model(self, image): if self.sca_filepath is None: self.logger.warning("No saturation data file provided; no saturation effect will be applied.") return image - + self.logger.warning("Lab measured model will be applied for saturation effect.") self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - - saturation_array = self.df['SATURATE'][:,:] #4096x4096 array + + saturation_array = self.df['SATURATE'][:, :] # 4096x4096 array where_sat = np.where(image.array > saturation_array) - image.array[ where_sat ] = saturation_array[ where_sat ] - return image \ No newline at end of file + image.array[where_sat] = saturation_array[where_sat] + return image diff --git a/roman_imsim/effects/setup_sky.py b/roman_imsim/effects/setup_sky.py index 0e128518..f2581099 100644 --- a/roman_imsim/effects/setup_sky.py +++ b/roman_imsim/effects/setup_sky.py @@ -1,46 +1,47 @@ import numpy as np import galsim import roman_imsim.effects as effects -from .utils import sca_number_to_file, get_pointing +from .utils import get_pointing + class setup_sky(object): def __init__(self, base, logger, rng, rng_iter=None): self.base = base self.logger = logger self.rng = rng - - self.visit = int(self.base['input']['obseq_data']['visit']) - self.sca = base['image']['SCA'] - self.pointing = get_pointing(self.base, self.visit, self.sca) + + self.visit = int(self.base['input']['obseq_data']['visit']) + self.sca = base['image']['SCA'] + self.pointing = get_pointing(self.base, self.visit, self.sca) def get_sky_image(self): - bounds = galsim.BoundsI( xmin=1, ymin=1, - xmax=4088, ymax=4088) + bounds = galsim.BoundsI(xmin=1, ymin=1, + xmax=4088, ymax=4088) self.sky_img = galsim.Image(bounds=bounds, wcs=self.pointing.WCS) self.pointing.WCS.makeSkyImage(self.sky_img, 0.) - - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) + + bound_pad = galsim.BoundsI(xmin=1, ymin=1, + xmax=4096, ymax=4096) im_pad = galsim.Image(bound_pad) - im_pad.array[4:-4, 4:-4] = self.sky_img.array[:,:] - + im_pad.array[4:-4, 4:-4] = self.sky_img.array[:, :] + effects_list = list(self.base['image']['add_effects']) if 'background' not in effects_list: return self.sky_img - + bkg_idx = effects_list.index('background') for i in range(bkg_idx, len(effects_list)): effect_name = effects_list[i] args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) effect = getattr(effects, effect_name)(*args) - im_pad = effect.apply(image = im_pad) - + im_pad = effect.apply(image=im_pad) + im_pad.quantize() # output 4088x4088 img in uint16 self.sky_img = im_pad.array[4:-4, 4:-4] self.sky_img = galsim.Image(self.sky_img, dtype=np.uint16) return self.sky_img - + def save_sky_img(self, outdir='.', sky_img_name='sky_img.fits'): - self.sky_img.write(sky_img_name, dir=outdir) \ No newline at end of file + self.sky_img.write(sky_img_name, dir=outdir) diff --git a/roman_imsim/effects/utils.py b/roman_imsim/effects/utils.py index fb3c9ac3..bd7834d1 100644 --- a/roman_imsim/effects/utils.py +++ b/roman_imsim/effects/utils.py @@ -1,32 +1,35 @@ import galsim as galsim import galsim.roman as roman +import numpy as np from roman_imsim.obseq import ObSeqDataLoader sca_number_to_file = { - 1 : 'SCA_22066_211227_v001.fits', - 2 : 'SCA_21815_211221_v001.fits', - 3 : 'SCA_21946_211225_v001.fits', - 4 : 'SCA_22073_211229_v001.fits', - 5 : 'SCA_21816_211222_v001.fits', - 6 : 'SCA_20663_211102_v001.fits', - 7 : 'SCA_22069_211228_v001.fits', - 8 : 'SCA_21641_211216_v001.fits', - 9 : 'SCA_21813_211219_v001.fits', - 10 : 'SCA_22078_211230_v001.fits', - 11 : 'SCA_21947_211226_v001.fits', - 12 : 'SCA_22077_211230_v001.fits', - 13 : 'SCA_22067_211227_v001.fits', - 14 : 'SCA_21814_211220_v001.fits', - 15 : 'SCA_21645_211228_v001.fits', - 16 : 'SCA_21643_211218_v001.fits', - 17 : 'SCA_21319_211211_v001.fits', - 18 : 'SCA_20833_211116_v001.fits', - } + 1 : 'SCA_22066_211227_v001.fits', + 2 : 'SCA_21815_211221_v001.fits', + 3 : 'SCA_21946_211225_v001.fits', + 4 : 'SCA_22073_211229_v001.fits', + 5 : 'SCA_21816_211222_v001.fits', + 6 : 'SCA_20663_211102_v001.fits', + 7 : 'SCA_22069_211228_v001.fits', + 8 : 'SCA_21641_211216_v001.fits', + 9 : 'SCA_21813_211219_v001.fits', + 10 : 'SCA_22078_211230_v001.fits', + 11 : 'SCA_21947_211226_v001.fits', + 12 : 'SCA_22077_211230_v001.fits', + 13 : 'SCA_22067_211227_v001.fits', + 14 : 'SCA_21814_211220_v001.fits', + 15 : 'SCA_21645_211228_v001.fits', + 16 : 'SCA_21643_211218_v001.fits', + 17 : 'SCA_21319_211211_v001.fits', + 18 : 'SCA_20833_211116_v001.fits', +} + class get_pointing(object): """ Class to store stuff about the telescope """ + def __init__(self, params, visit, SCA): self.params = params @@ -38,18 +41,19 @@ def __init__(self, params, visit, SCA): self.date = obseq_data.ob['date'] self.exptime = obseq_data.ob['exptime'] self.bpass = roman.getBandpasses()[self.filter] - self.WCS = roman.getWCS(world_pos = galsim.CelestialCoord(ra=obseq_data.ob['ra'], \ - dec=obseq_data.ob['dec']), - PA = obseq_data.ob['pa'], - date = self.date, - SCAs = self.sca, - PA_is_FPA = True + self.WCS = roman.getWCS(world_pos=galsim.CelestialCoord(ra=obseq_data.ob['ra'], + dec=obseq_data.ob['dec']), + PA=obseq_data.ob['pa'], + date=self.date, + SCAs=self.sca, + PA_is_FPA=True )[self.sca] - self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2),int(roman.n_pix/2))) - + self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2), int(roman.n_pix/2))) + + def translate_cvz(orig_radec, field_ra=9.5, field_dec=-44, cvz_ra=61.24, cvz_dec=-48.42): ra = orig_radec.ra/galsim.degrees-field_ra dec = orig_radec.dec/galsim.degrees-field_dec ra += cvz_ra / np.cos(cvz_dec*np.pi/180) dec += cvz_dec - return galsim.CelestialCoord(ra*galsim.degrees,dec*galsim.degrees) \ No newline at end of file + return galsim.CelestialCoord(ra*galsim.degrees, dec*galsim.degrees) diff --git a/roman_imsim/effects/vtpe.py b/roman_imsim/effects/vtpe.py index 0b5af82e..49d55a4f 100644 --- a/roman_imsim/effects/vtpe.py +++ b/roman_imsim/effects/vtpe.py @@ -1,20 +1,18 @@ import os import numpy as np import fitsio as fio -import galsim -import galsim.roman as roman from roman_imsim.effects import roman_effects -import roman_imsim.effects as effects -from galsim.config import ParseValue -from .utils import sca_number_to_file, get_pointing +from .utils import sca_number_to_file + class vtpe(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - + self.model = getattr(self, self.params['model']) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%(str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( + str(self.params['model']), str(self.__class__.__name__))) self.model = self.simple_model def simple_model(self, image): @@ -39,23 +37,23 @@ def lab_model(self, image): self.logger.warning("No VTPE data file provided; no VTPE will be applied.") return image self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - + self.logger.warning("Lab measured model will be applied for VTPE.") # expand 512x512 arrays to 4096x4096 t = np.zeros((4096, 512)) for row in range(t.shape[0]): - t[row, row//8] =1 - a_vtpe = t.dot(self.df['VTPE'][0,:,:][0]).dot(t.T) - ## NaN check + t[row, row//8] = 1 + a_vtpe = t.dot(self.df['VTPE'][0, :, :][0]).dot(t.T) + # NaN check if np.isnan(a_vtpe).any(): self.logger.warning("vtpe skipped due to NaN in file") return image - b_vtpe = t.dot(self.df['VTPE'][1,:,:][0]).dot(t.T) - dQ0 = t.dot(self.df['VTPE'][2,:,:][0]).dot(t.T) + b_vtpe = t.dot(self.df['VTPE'][1, :, :][0]).dot(t.T) + dQ0 = t.dot(self.df['VTPE'][2, :, :][0]).dot(t.T) dQ = image.array - np.roll(image.array, 1, axis=0) - dQ[0,:] *= 0 + dQ[0, :] *= 0 - image.array[:,:] += dQ * ( a_vtpe + b_vtpe * np.log( 1. + np.abs(dQ)/dQ0 )) - return image \ No newline at end of file + image.array[:, :] += dQ * (a_vtpe + b_vtpe * np.log(1. + np.abs(dQ)/dQ0)) + return image diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 6159ae36..7acc5648 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -6,16 +6,11 @@ from galsim.config import RegisterImageType from galsim.config.image_scattered import ScatteredImageBuilder from galsim.image import Image -from astropy.time import Time -import numpy as np -from astropy.time import Time -import numpy as np - -from .detector_effects import detector_effects import roman_imsim.effects as roman_effects + class RomanSCAImageBuilder(ScatteredImageBuilder): def setup(self, config, base, image_num, obj_num, ignore, logger): @@ -87,8 +82,8 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.mjd = params["mjd"] self.exptime = params["exptime"] - self.ignore_noise = params.get('ignore_noise', False) - + self.ignore_noise = params.get("ignore_noise", False) + # self.exptime = params.get('exptime', roman.exptime) # Default is roman standard exposure time. self.stray_light = params.get('stray_light', False) self.thermal_background = params.get('thermal_background', False) @@ -121,19 +116,11 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.visit = int(base['input']['obseq_data']['visit']) self.sca_filepath = params.get('sca_filepath', None) - self.effects = detector_effects(params = base, - visit = self.visit, - sca = self.sca, - filter = self.filter, - logger = logger, - rng = self.rng, - rng_iter = self.visit * self.sca, - sca_filepath = self.sca_filepath) # If user hasn't overridden the bandpass to use, get the standard one. - if 'bandpass' not in config: - base['bandpass'] = galsim.config.BuildBandpass(base['image'], 'bandpass', base, logger=logger) - + if "bandpass" not in config: + base["bandpass"] = galsim.config.BuildBandpass(base["image"], "bandpass", base, logger=logger) + self.base = base self.logger = logger @@ -231,7 +218,7 @@ def buildImage(self, config, base, image_num, obj_num, logger): ) logger.debug("image %d: Overlap = %s", image_num, str(bounds)) full_image[bounds] += stamps[k][bounds] - stamps=None + stamps = None # # [TODO] # break @@ -242,7 +229,6 @@ def buildImage(self, config, base, image_num, obj_num, logger): return full_image, None - def addNoise(self, image, config, base, image_num, obj_num, current_var, logger): """Add the final noise to a Scattered image @@ -260,165 +246,24 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) return base['current_noise_image'] = base['current_image'] - wcs = base['wcs'] - bp = base['bandpass'] # rng = galsim.config.GetRNG(config, base) - logger.info('image %d: Start RomanSCA detector effects',base.get('image_num',0)) - - - # self.effects.setup_sky(image, force_cvz=self.effects.force_cvz, stray_light=self.stray_light, thermal_background=self.thermal_background) - # [TODO] quantize() at this step? - - # The image up to here is an expectation value. - # Realize it as an integer number of photons. - # poisson_noise = galsim.noise.PoissonNoise(self.rng) - # if self.draw_method == 'phot': - # logger.debug("Adding poisson noise to sky photons") - # sky_image1 = sky_image.copy() - # sky_image1.addNoise(poisson_noise) - # image.quantize() # In case any profiles used InterpolatedImage, in which case - # # the image won't necessarily be integers. - # image += sky_image1 - # else: - # logger.debug("Adding poisson noise") - # image += sky_image - # image.addNoise(poisson_noise) - # image = self.effects.add_background(image, draw_method=self.draw_method) - + logger.info('image %d: Start RomanSCA detector effects', base.get('image_num', 0)) + # create padded image - bound_pad = galsim.BoundsI( xmin=1, ymin=1, - xmax=4096, ymax=4096) + bound_pad = galsim.BoundsI(xmin=1, ymin=1, + xmax=4096, ymax=4096) im_pad = galsim.Image(bound_pad) - im_pad.array[4:-4, 4:-4] = image.array[:,:] - + im_pad.array[4:-4, 4:-4] = image.array[:, :] + effects_list = self.base['image']['add_effects'].keys() for effect_name in effects_list: args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) effect = getattr(roman_effects, effect_name)(*args) - im_pad = effect.apply(image = im_pad) - + im_pad = effect.apply(image=im_pad) + im_pad.quantize() # output 4088x4088 img in uint16 - image.array[:,:] = im_pad.array[4:-4, 4:-4] - - # if self.sca_filepath is not None: - # ## create padded image - # bound_pad = galsim.BoundsI( xmin=1, ymin=1, - # xmax=4096, ymax=4096) - # im_pad = galsim.Image(bound_pad) - # im_pad.array[4:-4, 4:-4] = image.array[:,:] - # self.effects.set_diff(im_pad) - - # # im_pad = self.effects.qe(im_pad) - # # self.effects.diff('qe', im_pad) - # if self.qe: - # qe = self.effects.quantum_efficiency( - # params=self.base, - # logger=logger, - # model=self.qe, - # sca_filepath=self.sca_filepath - # ) - # im_pad = qe.apply(image = im_pad) - # self.effects.diff('qe', im_pad) - - # # im_pad = self.effects.bfe(im_pad) - # # self.effects.diff('bfe', im_pad) - # if self.bfe: - # # bfe = self.effects.bfe( - # # params=self.base, - # # logger=logger, - # # model=self.bfe, - # # sca_filepath=self.sca_filepath - # # ) - # bfe = roman_effects.bfe( - # params = self.base['image']['add_effects']['brighter_fatter'], - # base = self.base, - # logger = self.logger, - # rng = self.rng, - # ) - # im_pad = bfe.apply(image = im_pad) - # self.effects.diff('bfe', im_pad) - - # im_pad = self.effects.add_persistence(im_pad) - # self.effects.diff('pers', im_pad) - - # im_pad.quantize() - # self.effects.diff('quantize1', im_pad) - - # im_pad = self.effects.dark_current(im_pad) - # self.effects.diff('dark', im_pad) - - # im_pad = self.effects.saturate(im_pad) - # self.effects.diff('sat', im_pad) - - # # im_pad = self.effects.nonlinearity(im_pad) - # # self.effects.diff('cnl', im_pad) - - # if self.nonlinearity: - # nonlinearity = roman_effects.nonlinearity( - # params = self.base['image']['add_effects']['nonlinearity'], - # base = self.base, - # logger = self.logger, - # rng = self.rng, - # ) - # im_pad = nonlinearity.apply(image = im_pad) - # self.effects.diff('cnl', im_pad) - - # im_pad = self.effects.interpix_cap(im_pad) - # self.effects.diff('ipc', im_pad) - - # im_pad = self.effects.deadpix(im_pad) - # self.effects.diff('deadpix', im_pad) - - # im_pad = self.effects.vtpe(im_pad) - # self.effects.diff('vtpe', im_pad) - - # im_pad = self.effects.add_read_noise(im_pad) - # self.effects.diff('read', im_pad) - - # im_pad = self.effects.add_gain(im_pad) - # self.effects.diff('gain', im_pad) - - # im_pad = self.effects.add_bias(im_pad) - # self.effects.diff('bias', im_pad) - - # im_pad.quantize() - # self.effects.diff('quantize2', im_pad) - - # # output 4088x4088 img in uint16 - # image.array[:,:] = im_pad.array[4:-4, 4:-4] - - # # [TODO] - # # # data quality image - # # # 0x1 -> non-responsive - # # # 0x2 -> hot pixel - # # # 0x4 -> very hot pixel - # # # 0x8 -> adjacent to pixel with strange response - # # # 0x10 -> low CDS, high total noise pixel (may have strange settling behaviors, not recommended for precision applications) - # # # 0x20 -> CNL fit went down to the minimum number of points (remaining degrees of freedom = 0) - # # # 0x40 -> no solid-waffle solution for this region (set gain value to array median). normally occurs in a few small regions of some SCAs with lots of bad pixels. [recommend not to use these regions for WL analysis] - # # # 0x80 -> wt==0 - # # dq = self.df['BADPIX'][4:4092, 4:4092] - # # # get weight map - # # if wt is not None: - # # dq[wt==0] += 128 - - # # sky_noise = self.sky.copy() - # # sky_noise = self.finalize_sky_im(sky_noise, pointing) - - # else: - # image = self.effects.recip_failure(image) # Introduce reciprocity failure to image - # image.quantize() # At this point in the image generation process, an integer number of photons gets detected - # image = self.effects.dark_current(image) # Add dark current to image - # image = self.effects.add_persistence(image) - # image = self.effects.saturate(image) - # image= self.effects.nonlinearity(image) # Apply nonlinearity - # image = self.effects.interpix_cap(image) # Introduce interpixel capacitance to image. - # image = self.effects.add_read_noise(image) - # image = self.effects.e_to_ADU(image) # Convert electrons to ADU - - # Make integer ADU now. - # image.quantize() + image.array[:, :] = im_pad.array[4:-4, 4:-4] if self.sky_subtract: logger.debug("Subtracting sky image") @@ -426,7 +271,6 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) sky_image = sky.get_sky_image() image -= sky_image sky.save_sky_img(outdir=self.base['output']['dir']) - # Register this as a valid type From bb9168a74a2a949f0d69b349fe4e76dfcbfd23e8 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 29 Jul 2025 18:29:35 -0400 Subject: [PATCH 18/26] black reformated --- roman_imsim/__init__.py | 2 +- roman_imsim/effects/__init__.py | 2 +- roman_imsim/effects/background.py | 27 ++-- roman_imsim/effects/bias.py | 10 +- roman_imsim/effects/brighter_fatter.py | 179 ++++++++++++++-------- roman_imsim/effects/dark_current.py | 10 +- roman_imsim/effects/dead_pix.py | 10 +- roman_imsim/effects/gain.py | 12 +- roman_imsim/effects/interpix_cap.py | 35 +++-- roman_imsim/effects/nonlinearity.py | 35 +++-- roman_imsim/effects/persistence.py | 95 ++++++------ roman_imsim/effects/quantum_efficiency.py | 10 +- roman_imsim/effects/read_noise.py | 12 +- roman_imsim/effects/recip_failure.py | 12 +- roman_imsim/effects/roman_effects.py | 37 ++--- roman_imsim/effects/saturate.py | 15 +- roman_imsim/effects/setup_sky.py | 22 ++- roman_imsim/effects/utils.py | 72 ++++----- roman_imsim/effects/vtpe.py | 18 ++- 19 files changed, 351 insertions(+), 264 deletions(-) diff --git a/roman_imsim/__init__.py b/roman_imsim/__init__.py index dfba337a..2cbd35d0 100644 --- a/roman_imsim/__init__.py +++ b/roman_imsim/__init__.py @@ -18,4 +18,4 @@ from .wcs import * from .skycat import * from .photonOps import * -from .bandpass import * \ No newline at end of file +from .bandpass import * diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py index f91a8d6d..3ff4da12 100644 --- a/roman_imsim/effects/__init__.py +++ b/roman_imsim/effects/__init__.py @@ -13,4 +13,4 @@ from .read_noise import read_noise from .gain import gain from .bias import bias -from .setup_sky import setup_sky \ No newline at end of file +from .setup_sky import setup_sky diff --git a/roman_imsim/effects/background.py b/roman_imsim/effects/background.py index 260f1b1e..f940d029 100644 --- a/roman_imsim/effects/background.py +++ b/roman_imsim/effects/background.py @@ -7,14 +7,17 @@ class background(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.thermal_background = self.params['thermal_background'] if 'thermal_background' \ - in self.params else False - self.stray_light = self.params['stray_light'] if 'stray_light' in self.params else False + self.thermal_background = ( + self.params["thermal_background"] if "thermal_background" in self.params else False + ) + self.stray_light = self.params["stray_light"] if "stray_light" in self.params else False - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -29,23 +32,23 @@ def simple_model(self, image): else: radec = pointing.radec sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) - self.logger.debug('Adding sky_level = %s', sky_level) + self.logger.debug("Adding sky_level = %s", sky_level) if self.stray_light: - self.logger.debug('Stray light fraction = %s', roman.stray_light_fraction) - sky_level *= (1.0 + roman.stray_light_fraction) + self.logger.debug("Stray light fraction = %s", roman.stray_light_fraction) + sky_level *= 1.0 + roman.stray_light_fraction # Create sky image self.sky = galsim.Image(bounds=image.bounds, wcs=pointing.WCS) pointing.WCS.makeSkyImage(self.sky, sky_level) if self.thermal_background: tb = roman.thermal_backgrounds[pointing.filter] * pointing.exptime - self.logger.debug('Adding thermal background: %s', tb) + self.logger.debug("Adding thermal background: %s", tb) self.sky += tb self.sky.addNoise(self.noise) # [TODO] Not entirely sure about this block, since the 'auto' option is meant to # let the software choose which drawing method to use based on the total flux. - if self.base['image']['draw_method'] not in ['phot', 'auto']: + if self.base["image"]["draw_method"] not in ["phot", "auto"]: image.addNoise(self.noise) # Adding sky level to the image. @@ -53,5 +56,5 @@ def simple_model(self, image): if self.save_diff: prev = image.copy() diff = prev - orig - diff.write(os.path.join(self.diff_dir, 'sky_a.fits')) + diff.write(os.path.join(self.diff_dir, "sky_a.fits")) return image diff --git a/roman_imsim/effects/bias.py b/roman_imsim/effects/bias.py index f4ce606f..eb6a5db9 100644 --- a/roman_imsim/effects/bias.py +++ b/roman_imsim/effects/bias.py @@ -8,10 +8,12 @@ class bias(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -25,7 +27,7 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for bias.") - bias = self.df['BIAS'][:, :] # 4096x4096 img + bias = self.df["BIAS"][:, :] # 4096x4096 img image.array[:, :] += bias return image diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/effects/brighter_fatter.py index cfa18b14..9b334a99 100644 --- a/roman_imsim/effects/brighter_fatter.py +++ b/roman_imsim/effects/brighter_fatter.py @@ -9,10 +9,12 @@ class brighter_fatter(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -50,14 +52,14 @@ def lab_model(self, image): n_max = 32 m_max = 32 num_grids = 4 - n_sub = n_max//num_grids - m_sub = m_max//num_grids + n_sub = n_max // num_grids + m_sub = m_max // num_grids # ======================================================================= # solve boundary shfit kernel aX components # ======================================================================= - a_area = self.df['BFE'][:, :, :, :] # 5x5x32x32 - a_components = np.zeros((4, 2*nbfe+1, 2*nbfe+1, n_max, m_max)) # 4x5x5x32x32 + a_area = self.df["BFE"][:, :, :, :] # 5x5x32x32 + a_components = np.zeros((4, 2 * nbfe + 1, 2 * nbfe + 1, n_max, m_max)) # 4x5x5x32x32 # solve aR aT aL aB for each a for n in range(n_max): # m_max and n_max = 32 (binned in 128x128) @@ -65,34 +67,45 @@ def lab_model(self, image): a = a_area[:, :, n, m] # a in (2 x nbfe+1)*(2 x nbfe+1) # assume two parity symmetries - a = (a + np.fliplr(a) + np.flipud(a) + np.flip(a))/4. - - r = 0.5 * (3.25/4.25)**(1.5) / 1.5 # source-boundary projection - B = (a[2, 2], a[3, 2], a[2, 3], a[3, 3], - a[4, 2], a[2, 4], a[3, 4], a[4, 4]) - - A = np.array([[-2 , -2 , 0 , 0 , 0 , 0 , 0], - [0 , 1 , 0 , -1 , -2 , 0 , 0], - [1 , 0 , -1 , 0 , -2 , 0 , 0], - [0 , 0 , 0 , 0 , 2 , -2 , 0], - [0 , 0 , 0 , 1 , 0 , -2*r, 0], - [0 , 0 , 1 , 0 , 0 , -2*r, 0], - [0 , 0 , 0 , 0 , 0 , 1+r, -1], - [0 , 0 , 0 , 0 , 0 , 0 , 2]]) + a = (a + np.fliplr(a) + np.flipud(a) + np.flip(a)) / 4.0 + + r = 0.5 * (3.25 / 4.25) ** (1.5) / 1.5 # source-boundary projection + B = (a[2, 2], a[3, 2], a[2, 3], a[3, 3], a[4, 2], a[2, 4], a[3, 4], a[4, 4]) + + A = np.array( + [ + [-2, -2, 0, 0, 0, 0, 0], + [0, 1, 0, -1, -2, 0, 0], + [1, 0, -1, 0, -2, 0, 0], + [0, 0, 0, 0, 2, -2, 0], + [0, 0, 0, 1, 0, -2 * r, 0], + [0, 0, 1, 0, 0, -2 * r, 0], + [0, 0, 0, 0, 0, 1 + r, -1], + [0, 0, 0, 0, 0, 0, 2], + ] + ) s1, s2, s3, s4, s5, s6, s7 = np.linalg.lstsq(A, B, rcond=None)[0] - aR = np.array([[0. , -s7 , -r*s6 , r*s6 , s7], - [0. , -s6 , -s5 , s5 , s6], - [0. , -s3 , -s1 , s1 , s3], - [0. , -s6 , -s5 , s5 , s6], - [0. , -s7 , -r*s6 , r*s6 , s7],]) - - aT = np.array([[0. , 0. , 0. , 0. , 0.], - [-s7 , -s6 , -s4 , -s6 , -s7], - [-r*s6 , -s5 , -s2 , -s5 , -r*s6], - [r*s6 , s5 , s2 , s5 , r*s6], - [s7 , s6 , s4 , s6 , s7],]) + aR = np.array( + [ + [0.0, -s7, -r * s6, r * s6, s7], + [0.0, -s6, -s5, s5, s6], + [0.0, -s3, -s1, s1, s3], + [0.0, -s6, -s5, s5, s6], + [0.0, -s7, -r * s6, r * s6, s7], + ] + ) + + aT = np.array( + [ + [0.0, 0.0, 0.0, 0.0, 0.0], + [-s7, -s6, -s4, -s6, -s7], + [-r * s6, -s5, -s2, -s5, -r * s6], + [r * s6, s5, s2, s5, r * s6], + [s7, s6, s4, s6, s7], + ] + ) aL = aR[::-1, ::-1] aB = aT[::-1, ::-1] @@ -115,58 +128,96 @@ def lab_model(self, image): # where_sat = np.where(array_pad > saturation_array) # array_pad[ where_sat ] = saturation_array[ where_sat ] # array_pad = array_pad[4:-4,4:-4] - saturate = self.cross_refer('saturate') + saturate = self.cross_refer("saturate") array_pad = saturate.apply(image=image.copy()).array[4:-4, 4:-4] # img of interest 4088x4088 - array_pad = np.pad(array_pad, [(4+nbfe, 4+nbfe), (4+nbfe, 4+nbfe)], - mode='symmetric') # 4100x4100 array + array_pad = np.pad( + array_pad, [(4 + nbfe, 4 + nbfe), (4 + nbfe, 4 + nbfe)], mode="symmetric" + ) # 4100x4100 array # (4, 4096, 4096) in order of [aR, aT, aL, aB] - dQ_components = np.zeros((4, bin_size*n_max, bin_size*m_max)) + dQ_components = np.zeros((4, bin_size * n_max, bin_size * m_max)) # run in sub grids to reduce memory # pad and expand kernels - t = np.zeros((bin_size*n_sub, n_sub)) + t = np.zeros((bin_size * n_sub, n_sub)) for row in range(t.shape[0]): - t[row, row//(bin_size)] = 1 + t[row, row // (bin_size)] = 1 for gj in range(num_grids): for gi in range(num_grids): # (4,5,5,sub_grid,sub_grid) - a_components_pad = np.zeros((4, 2*nbfe+1, 2*nbfe+1, bin_size - * n_sub+2*nbfe, bin_size*m_sub+2*nbfe)) + a_components_pad = np.zeros( + (4, 2 * nbfe + 1, 2 * nbfe + 1, bin_size * n_sub + 2 * nbfe, bin_size * m_sub + 2 * nbfe) + ) for comp in range(4): - for j in range(2*nbfe+1): - for i in range(2*nbfe+1): + for j in range(2 * nbfe + 1): + for i in range(2 * nbfe + 1): # sub_grid*sub_grid - tmp = (t.dot(a_components[comp, j, i, gj*n_sub:(gj+1) - * n_sub, gi*m_sub:(gi+1)*m_sub])).dot(t.T) + tmp = ( + t.dot( + a_components[ + comp, + j, + i, + gj * n_sub : (gj + 1) * n_sub, + gi * m_sub : (gi + 1) * m_sub, + ] + ) + ).dot(t.T) a_components_pad[comp, j, i, :, :] = np.pad( - tmp, [(nbfe, nbfe), (nbfe, nbfe)], mode='symmetric') + tmp, [(nbfe, nbfe), (nbfe, nbfe)], mode="symmetric" + ) # convolve aX_ij with Q_ij for comp in range(4): - for dy in range(-nbfe, nbfe+1): - for dx in range(-nbfe, nbfe+1): - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub, - gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - += a_components_pad[comp, nbfe+dy, nbfe+dx, nbfe-dy:nbfe-dy+bin_size*n_sub, - nbfe-dx:nbfe-dx+bin_size*m_sub]\ - * array_pad[-dy + nbfe + gj*bin_size*n_sub : -dy + nbfe - + (gj+1)*bin_size*n_sub, -dx + nbfe + gi*bin_size*m_sub : -dx - + nbfe + (gi+1)*bin_size*m_sub] - - dj = int(np.sin(comp*np.pi/2)) - di = int(np.cos(comp*np.pi/2)) - - dQ_components[comp, gj*bin_size*n_sub : (gj+1)*bin_size*n_sub , - gi*bin_size*m_sub : (gi+1)*bin_size*m_sub]\ - *= 0.5*(array_pad[nbfe + gj*bin_size*n_sub : nbfe + (gj+1)*bin_size*n_sub, - nbfe + gi*bin_size*m_sub : nbfe + (gi+1)*bin_size*m_sub] - + array_pad[dj+nbfe + gj*bin_size*n_sub : dj+nbfe + (gj+1)*bin_size*n_sub - , di+nbfe + gi*bin_size*m_sub : di+nbfe + (gi+1)*bin_size*m_sub]) + for dy in range(-nbfe, nbfe + 1): + for dx in range(-nbfe, nbfe + 1): + dQ_components[ + comp, + gj * bin_size * n_sub : (gj + 1) * bin_size * n_sub, + gi * bin_size * m_sub : (gi + 1) * bin_size * m_sub, + ] += ( + a_components_pad[ + comp, + nbfe + dy, + nbfe + dx, + nbfe - dy : nbfe - dy + bin_size * n_sub, + nbfe - dx : nbfe - dx + bin_size * m_sub, + ] + * array_pad[ + -dy + + nbfe + + gj * bin_size * n_sub : -dy + + nbfe + + (gj + 1) * bin_size * n_sub, + -dx + + nbfe + + gi * bin_size * m_sub : -dx + + nbfe + + (gi + 1) * bin_size * m_sub, + ] + ) + + dj = int(np.sin(comp * np.pi / 2)) + di = int(np.cos(comp * np.pi / 2)) + + dQ_components[ + comp, + gj * bin_size * n_sub : (gj + 1) * bin_size * n_sub, + gi * bin_size * m_sub : (gi + 1) * bin_size * m_sub, + ] *= 0.5 * ( + array_pad[ + nbfe + gj * bin_size * n_sub : nbfe + (gj + 1) * bin_size * n_sub, + nbfe + gi * bin_size * m_sub : nbfe + (gi + 1) * bin_size * m_sub, + ] + + array_pad[ + dj + nbfe + gj * bin_size * n_sub : dj + nbfe + (gj + 1) * bin_size * n_sub, + di + nbfe + gi * bin_size * m_sub : di + nbfe + (gi + 1) * bin_size * m_sub, + ] + ) image.array[:, :] -= dQ_components.sum(axis=0) image.array[:, 1:] += dQ_components[0][:, :-1] diff --git a/roman_imsim/effects/dark_current.py b/roman_imsim/effects/dark_current.py index bdcc417d..77032468 100644 --- a/roman_imsim/effects/dark_current.py +++ b/roman_imsim/effects/dark_current.py @@ -10,10 +10,12 @@ class dark_current(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -35,7 +37,7 @@ def lab_model(self, image): exptime = self.pointing.exptime self.logger.warning("Lab measured model will be applied for dark current.") - self.dark_current_ = roman.dark_current * exptime + self.df['DARK'][:, :].flatten() * exptime + self.dark_current_ = roman.dark_current * exptime + self.df["DARK"][:, :].flatten() * exptime dark_current_ = self.dark_current_.clip(0) # opt for numpy random geneator instead for speed self.im_dark = self.rng_np.poisson(dark_current_).reshape(image.array.shape).astype(image.dtype) diff --git a/roman_imsim/effects/dead_pix.py b/roman_imsim/effects/dead_pix.py index 69332310..2580fb14 100644 --- a/roman_imsim/effects/dead_pix.py +++ b/roman_imsim/effects/dead_pix.py @@ -8,10 +8,12 @@ class dead_pix(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -25,7 +27,7 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for dead pixel.") - dead_mask = self.df['BADPIX'][:, :] & 1 # 4096x4096 array + dead_mask = self.df["BADPIX"][:, :] & 1 # 4096x4096 array image.array[dead_mask > 0] = 0 return image diff --git a/roman_imsim/effects/gain.py b/roman_imsim/effects/gain.py index 650210e8..e82fce68 100644 --- a/roman_imsim/effects/gain.py +++ b/roman_imsim/effects/gain.py @@ -10,10 +10,12 @@ class gain(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -29,11 +31,11 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for gain.") - gain_map = self.df['GAIN'][:, :] # 32x32 img + gain_map = self.df["GAIN"][:, :] # 32x32 img t = np.zeros((4096, 32)) for row in range(t.shape[0]): - t[row, row//128] = 1 + t[row, row // 128] = 1 gain_expand = (t.dot(gain_map)).dot(t.T) # 4096x4096 gain img image.array[:, :] /= gain_expand return image diff --git a/roman_imsim/effects/interpix_cap.py b/roman_imsim/effects/interpix_cap.py index aba6125e..4de17a06 100644 --- a/roman_imsim/effects/interpix_cap.py +++ b/roman_imsim/effects/interpix_cap.py @@ -10,16 +10,18 @@ class interpix_cap(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): self.logger.warning("Simple model will be applied for IPC effect.") kernel = roman.ipc_kernel - image.applyIPC(kernel, edge_treatment='extend', fill_value=None) + image.applyIPC(kernel, edge_treatment="extend", fill_value=None) return image def lab_model(self, image): @@ -31,35 +33,40 @@ def lab_model(self, image): self.logger.warning("Lab measured model will be applied for IPC effect.") # pad the array by one pixel at the four edges num_grids = 4 # num_grids <= 8 - grid_size = 4096//num_grids + grid_size = 4096 // num_grids array_pad = image.array[4:-4, 4:-4] # it's an array instead of img - array_pad = np.pad(array_pad, [(5, 5), (5, 5)], mode='symmetric') # 4098x4098 array + array_pad = np.pad(array_pad, [(5, 5), (5, 5)], mode="symmetric") # 4098x4098 array - K = self.df['IPC'][:, :, :, :] # 3,3,512, 512 + K = self.df["IPC"][:, :, :, :] # 3,3,512, 512 t = np.zeros((grid_size, 512)) for row in range(t.shape[0]): - t[row, row//(grid_size//512)] = 1 + t[row, row // (grid_size // 512)] = 1 array_out = np.zeros((4096, 4096)) # split job in sub_grids to reduce memory for gj in range(num_grids): for gi in range(num_grids): - K_pad = np.zeros((3, 3, grid_size+2, grid_size+2)) + K_pad = np.zeros((3, 3, grid_size + 2, grid_size + 2)) for j in range(3): for i in range(3): tmp = (t.dot(K[j, i, :, :])).dot(t.T) # grid_sizexgrid_size - K_pad[j, i, :, :] = np.pad(tmp, [(1, 1), (1, 1)], mode='symmetric') + K_pad[j, i, :, :] = np.pad(tmp, [(1, 1), (1, 1)], mode="symmetric") for dy in range(-1, 2): for dx in range(-1, 2): - array_out[gj*grid_size: (gj+1)*grid_size, gi*grid_size:(gi+1)*grid_size]\ - += K_pad[1+dy, 1+dx, 1-dy: 1-dy+grid_size, 1-dx:1-dx+grid_size] \ - * array_pad[1-dy+gj*grid_size: 1-dy+(gj+1)*grid_size, - 1-dx+gi*grid_size:1-dx+(gi+1)*grid_size] + array_out[ + gj * grid_size : (gj + 1) * grid_size, gi * grid_size : (gi + 1) * grid_size + ] += ( + K_pad[1 + dy, 1 + dx, 1 - dy : 1 - dy + grid_size, 1 - dx : 1 - dx + grid_size] + * array_pad[ + 1 - dy + gj * grid_size : 1 - dy + (gj + 1) * grid_size, + 1 - dx + gi * grid_size : 1 - dx + (gi + 1) * grid_size, + ] + ) image.array[:, :] = array_out return image diff --git a/roman_imsim/effects/nonlinearity.py b/roman_imsim/effects/nonlinearity.py index ef1cc96d..8be3e955 100644 --- a/roman_imsim/effects/nonlinearity.py +++ b/roman_imsim/effects/nonlinearity.py @@ -7,23 +7,25 @@ class nonlinearity(roman_effects): """ - Applying a quadratic non-linearity. - - Note that users who wish to apply some other nonlinearity function (perhaps for other NIR - detectors, or for CCDs) can use the more general nonlinearity routine, which uses the - following syntax: - final_image.applyNonlinearity(NLfunc=NLfunc) - with NLfunc being a callable function that specifies how the output image pixel values - should relate to the input ones. + Applying a quadratic non-linearity. + + Note that users who wish to apply some other nonlinearity function (perhaps for other NIR + detectors, or for CCDs) can use the more general nonlinearity routine, which uses the + following syntax: + final_image.applyNonlinearity(NLfunc=NLfunc) + with NLfunc being a callable function that specifies how the output image pixel values + should relate to the input ones. """ def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -34,15 +36,18 @@ def simple_model(self, image): def lab_model(self, image): if self.sca_filepath is None: self.logger.warning( - "No non-linearity data file provided; no non-linearity effect will be applied.") + "No non-linearity data file provided; no non-linearity effect will be applied." + ) return image self.logger.warning("Lab measured model will be applied for non-linearity effect.") self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - image.array[:, :] -= self.df['CNL'][0, :, :][0] * image.array**2 +\ - self.df['CNL'][1, :, :][0] * image.array**3 +\ - self.df['CNL'][2, :, :][0] * image.array**4 + image.array[:, :] -= ( + self.df["CNL"][0, :, :][0] * image.array**2 + + self.df["CNL"][1, :, :][0] * image.array**3 + + self.df["CNL"][2, :, :][0] * image.array**4 + ) return image diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/effects/persistence.py index 330f3ad7..17616765 100644 --- a/roman_imsim/effects/persistence.py +++ b/roman_imsim/effects/persistence.py @@ -10,25 +10,28 @@ class persistence(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model p_list = np.array([get_pointing(self.base, i, self.sca) for i in range(self.visit - 10, self.visit)]) dt_list = np.array([(self.pointing.date - p.date).total_seconds() for p in p_list]) - self.p_pers = p_list[np.where((dt_list > 0) & (dt_list < self.pointing.exptime*10))] + self.p_pers = p_list[np.where((dt_list > 0) & (dt_list < self.pointing.exptime * 10))] def simple_model(self, image): self.logger.warning("Simple model will be applied for persistence effect.") for p in self.p_pers: - dt = (self.pointing.date - p.date).total_seconds() - \ - self.pointing.exptime/2 # avg time since end of exposures + dt = ( + self.pointing.date - p.date + ).total_seconds() - self.pointing.exptime / 2 # avg time since end of exposures # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] - imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) - fn = os.path.join(self.base['output']['dir'], imfilename) + imfilename = self.base["output"]["file_name"]["format"] % (p.filter, p.visit, p.sca) + fn = os.path.join(self.base["output"]["dir"], imfilename) # [TODO] if not os.path.exists(fn): @@ -36,18 +39,18 @@ def simple_model(self, image): # apply all the effects that occured before persistence on the previouse exposures # since max of the sky background is of order 100, it is thus negligible for persistence - bound_pad = galsim.BoundsI(xmin=1, ymin=1, - xmax=4096, ymax=4096) + bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) x = galsim.Image(bound_pad) x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] - recip_failure = self.cross_refer('recip_failure') + recip_failure = self.cross_refer("recip_failure") x = recip_failure.apply(image=x) x.array.clip(0) # remove negative stimulus - image.array[:, - :] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * self.pointing.exptime + image.array[:, :] += ( + galsim.roman.roman_detectors.fermi_linear(x.array, dt) * self.pointing.exptime + ) return image @@ -60,25 +63,26 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) # setup parameters for persistence - Q01 = self.df['PERSIST'].read_header()['Q01'] - Q02 = self.df['PERSIST'].read_header()['Q02'] - Q03 = self.df['PERSIST'].read_header()['Q03'] - Q04 = self.df['PERSIST'].read_header()['Q04'] - Q05 = self.df['PERSIST'].read_header()['Q05'] - Q06 = self.df['PERSIST'].read_header()['Q06'] - alpha = self.df['PERSIST'].read_header()['ALPHA'] + Q01 = self.df["PERSIST"].read_header()["Q01"] + Q02 = self.df["PERSIST"].read_header()["Q02"] + Q03 = self.df["PERSIST"].read_header()["Q03"] + Q04 = self.df["PERSIST"].read_header()["Q04"] + Q05 = self.df["PERSIST"].read_header()["Q05"] + Q06 = self.df["PERSIST"].read_header()["Q06"] + alpha = self.df["PERSIST"].read_header()["ALPHA"] # iterate over previous exposures for p in self.p_pers: - dt = (self.pointing.date - p.date).total_seconds() - \ - self.pointing.exptime/2 # avg time since end of exposures + dt = ( + self.pointing.date - p.date + ).total_seconds() - self.pointing.exptime / 2 # avg time since end of exposures # linear time dependence (approximate until we get t1 and Delat t of the data) - fac_dt = (self.pointing.exptime/2.) / dt + fac_dt = (self.pointing.exptime / 2.0) / dt # self.base['output']['file_name']['items'] = [p.filter, p.visit, p.sca] # imfilename = ParseValue(self.base['output'], 'file_name', self.base, str)[0] - imfilename = self.base['output']['file_name']['format']%(p.filter, p.visit, p.sca) - fn = os.path.join(self.base['output']['dir'], imfilename) + imfilename = self.base["output"]["file_name"]["format"] % (p.filter, p.visit, p.sca) + fn = os.path.join(self.base["output"]["dir"], imfilename) # [TODO] if not os.path.exists(fn): @@ -87,46 +91,45 @@ def lab_model(self, image): # apply all the effects that occured before persistence on the previouse exposures # since max of the sky background is of order 100, it is thus negligible for persistence # same for brighter fatter effect - bound_pad = galsim.BoundsI(xmin=1, ymin=1, - xmax=4096, ymax=4096) + bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) x = galsim.Image(bound_pad) x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] # x = self.qe(x).array[:,:] - qe = self.cross_refer('quantum_efficiency') + qe = self.cross_refer("quantum_efficiency") x = qe.apply(image=x).array[:, :] x = x.clip(0.1) # remove negative and zero stimulus # Do linear interpolation a = np.zeros(x.shape) - a += ((x < Q01)) * x/Q01 - a += ((x >= Q01) & (x < Q02)) * (Q02-x)/(Q02-Q01) - image.array[:, :] += a * self.df['PERSIST'][0, :, :][0] * fac_dt + a += ((x < Q01)) * x / Q01 + a += ((x >= Q01) & (x < Q02)) * (Q02 - x) / (Q02 - Q01) + image.array[:, :] += a * self.df["PERSIST"][0, :, :][0] * fac_dt a = np.zeros(x.shape) - a += ((x >= Q01) & (x < Q02)) * (x-Q01)/(Q02-Q01) - a += ((x >= Q02) & (x < Q03)) * (Q03-x)/(Q03-Q02) - image.array[:, :] += a * self.df['PERSIST'][1, :, :][0] * fac_dt + a += ((x >= Q01) & (x < Q02)) * (x - Q01) / (Q02 - Q01) + a += ((x >= Q02) & (x < Q03)) * (Q03 - x) / (Q03 - Q02) + image.array[:, :] += a * self.df["PERSIST"][1, :, :][0] * fac_dt a = np.zeros(x.shape) - a += ((x >= Q02) & (x < Q03)) * (x-Q02)/(Q03-Q02) - a += ((x >= Q03) & (x < Q04)) * (Q04-x)/(Q04-Q03) - image.array[:, :] += a * self.df['PERSIST'][2, :, :][0] * fac_dt + a += ((x >= Q02) & (x < Q03)) * (x - Q02) / (Q03 - Q02) + a += ((x >= Q03) & (x < Q04)) * (Q04 - x) / (Q04 - Q03) + image.array[:, :] += a * self.df["PERSIST"][2, :, :][0] * fac_dt a = np.zeros(x.shape) - a += ((x >= Q03) & (x < Q04)) * (x-Q03)/(Q04-Q03) - a += ((x >= Q04) & (x < Q05)) * (Q05-x)/(Q05-Q04) - image.array[:, :] += a * self.df['PERSIST'][3, :, :][0] * fac_dt + a += ((x >= Q03) & (x < Q04)) * (x - Q03) / (Q04 - Q03) + a += ((x >= Q04) & (x < Q05)) * (Q05 - x) / (Q05 - Q04) + image.array[:, :] += a * self.df["PERSIST"][3, :, :][0] * fac_dt a = np.zeros(x.shape) - a += ((x >= Q04) & (x < Q05)) * (x-Q04)/(Q05-Q04) - a += ((x >= Q05) & (x < Q06)) * (Q06-x)/(Q06-Q05) - image.array[:, :] += a * self.df['PERSIST'][4, :, :][0] * fac_dt + a += ((x >= Q04) & (x < Q05)) * (x - Q04) / (Q05 - Q04) + a += ((x >= Q05) & (x < Q06)) * (Q06 - x) / (Q06 - Q05) + image.array[:, :] += a * self.df["PERSIST"][4, :, :][0] * fac_dt a = np.zeros(x.shape) - a += ((x >= Q05) & (x < Q06)) * (x-Q05)/(Q06-Q05) - a += ((x >= Q06)) * (x/Q06)**alpha # avoid fractional power of negative values - image.array[:, :] += a * self.df['PERSIST'][5, :, :][0] * fac_dt + a += ((x >= Q05) & (x < Q06)) * (x - Q05) / (Q06 - Q05) + a += ((x >= Q06)) * (x / Q06) ** alpha # avoid fractional power of negative values + image.array[:, :] += a * self.df["PERSIST"][5, :, :][0] * fac_dt return image diff --git a/roman_imsim/effects/quantum_efficiency.py b/roman_imsim/effects/quantum_efficiency.py index f175ef62..637b8766 100644 --- a/roman_imsim/effects/quantum_efficiency.py +++ b/roman_imsim/effects/quantum_efficiency.py @@ -8,10 +8,12 @@ class quantum_efficiency(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def lab_model(self, image): @@ -21,5 +23,5 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for quantum efficiency.") - image.array[:, :] *= self.df['RELQE1'][:, :] # 4096x4096 array + image.array[:, :] *= self.df["RELQE1"][:, :] # 4096x4096 array return image diff --git a/roman_imsim/effects/read_noise.py b/roman_imsim/effects/read_noise.py index dd6080c2..048dd43c 100644 --- a/roman_imsim/effects/read_noise.py +++ b/roman_imsim/effects/read_noise.py @@ -8,10 +8,12 @@ class read_noise(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -29,7 +31,7 @@ def lab_model(self, image): self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) self.logger.warning("Lab measured model will be applied for read noise.") - rdn = self.df['READ'][2, :, :].flatten() # flattened 4096x4096 array - self.im_read = self.rng_np.normal(loc=0., scale=rdn).reshape(image.array.shape).astype(image.dtype) + rdn = self.df["READ"][2, :, :].flatten() # flattened 4096x4096 array + self.im_read = self.rng_np.normal(loc=0.0, scale=rdn).reshape(image.array.shape).astype(image.dtype) image.array[:, :] += self.im_read return image diff --git a/roman_imsim/effects/recip_failure.py b/roman_imsim/effects/recip_failure.py index e0bc442b..8bc410d0 100644 --- a/roman_imsim/effects/recip_failure.py +++ b/roman_imsim/effects/recip_failure.py @@ -6,13 +6,15 @@ class recip_failure(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.alpha = self.params['alpha'] if 'alpha' in self.params else roman.reciprocity_alpha - self.base_flux = self.params['base_flux'] if 'base_flux' in self.params else 1.0 + self.alpha = self.params["alpha"] if "alpha" in self.params else roman.reciprocity_alpha + self.base_flux = self.params["base_flux"] if "base_flux" in self.params else 1.0 - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): diff --git a/roman_imsim/effects/roman_effects.py b/roman_imsim/effects/roman_effects.py index cadab6d0..1ff96837 100644 --- a/roman_imsim/effects/roman_effects.py +++ b/roman_imsim/effects/roman_effects.py @@ -12,10 +12,10 @@ class roman_effects(object): def __init__(self, params, base, logger, rng, rng_iter=None): self.params = params self.base = base - self.visit = int(self.base['input']['obseq_data']['visit']) - self.sca = base['image']['SCA'] - self.filter = base['image']['filter'] - self.sca_filepath = base['image']['sca_filepath'] + self.visit = int(self.base["input"]["obseq_data"]["visit"]) + self.sca = base["image"]["SCA"] + self.filter = base["image"]["filter"] + self.sca_filepath = base["image"]["sca_filepath"] self.rng_iter = rng_iter if rng_iter else self.visit * self.sca self.rng = rng @@ -26,34 +26,35 @@ def __init__(self, params, base, logger, rng, rng_iter=None): self.logger = logger self.force_cvz = False - if 'force_cvz' in self.base['image']['wcs']: - if self.base['image']['wcs']['force_cvz']: + if "force_cvz" in self.base["image"]["wcs"]: + if self.base["image"]["wcs"]["force_cvz"]: self.force_cvz = True self.save_diff = False - if 'save_diff' in self.base['image']: - self.save_diff = bool(self.base['image']['save_diff']) - if 'diff_dir' in self.base['output']: - self.diff_dir = self.base['output']['diff_dir'] + if "save_diff" in self.base["image"]: + self.save_diff = bool(self.base["image"]["save_diff"]) + if "diff_dir" in self.base["output"]: + self.diff_dir = self.base["output"]["diff_dir"] else: - self.diff_dir = self.base['output']['dir'] + self.diff_dir = self.base["output"]["dir"] def simple_model(self, image): self.logger.info("Applying the default model...") return image def cross_refer(self, effect_name): - if effect_name not in self.base['image']['add_effects']: + if effect_name not in self.base["image"]["add_effects"]: try: effect = getattr(effects, effect_name)( - {'model': 'simple_model'}, self.base, self.logger, self.rng) + {"model": "simple_model"}, self.base, self.logger, self.rng + ) except Exception as e: self.logger.warning(e) # self.logger.warning("Effect %s is not implemented!"%(effect_name)) return None else: try: - params = self.base['image']['add_effects'][effect_name] + params = self.base["image"]["add_effects"][effect_name] effect = getattr(effects, effect_name)(params, self.base, self.logger, self.rng) except Exception as e: self.logger.warning(e) @@ -68,13 +69,13 @@ def apply(self, image): def set_diff(self, im=None): if self.save_diff: self.pre = im.copy() - self.pre.write('bg.fits', dir=self.diff_dir) + self.pre.write("bg.fits", dir=self.diff_dir) return def diff(self, msg, im=None, verbose=True): if self.save_diff: - diff = im-self.pre - diff.write('%s_diff.fits'%msg , dir=self.diff_dir) + diff = im - self.pre + diff.write("%s_diff.fits" % msg, dir=self.diff_dir) self.pre = im.copy() - im.write('%s_cumul.fits'%msg, dir=self.diff_dir) + im.write("%s_cumul.fits" % msg, dir=self.diff_dir) return diff --git a/roman_imsim/effects/saturate.py b/roman_imsim/effects/saturate.py index 194d56df..01f29679 100644 --- a/roman_imsim/effects/saturate.py +++ b/roman_imsim/effects/saturate.py @@ -9,14 +9,17 @@ class saturate(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model - self.saturation_level = self.params['saturation_level'] \ - if 'saturation_level' in self.params else 100000 + self.saturation_level = ( + self.params["saturation_level"] if "saturation_level" in self.params else 100000 + ) def simple_model(self, image): self.logger.warning("Simple model will be applied for saturation.") @@ -33,7 +36,7 @@ def lab_model(self, image): self.logger.warning("Lab measured model will be applied for saturation effect.") self.df = fio.FITS(os.path.join(self.sca_filepath, sca_number_to_file[self.sca])) - saturation_array = self.df['SATURATE'][:, :] # 4096x4096 array + saturation_array = self.df["SATURATE"][:, :] # 4096x4096 array where_sat = np.where(image.array > saturation_array) image.array[where_sat] = saturation_array[where_sat] return image diff --git a/roman_imsim/effects/setup_sky.py b/roman_imsim/effects/setup_sky.py index f2581099..1da94895 100644 --- a/roman_imsim/effects/setup_sky.py +++ b/roman_imsim/effects/setup_sky.py @@ -10,29 +10,27 @@ def __init__(self, base, logger, rng, rng_iter=None): self.logger = logger self.rng = rng - self.visit = int(self.base['input']['obseq_data']['visit']) - self.sca = base['image']['SCA'] + self.visit = int(self.base["input"]["obseq_data"]["visit"]) + self.sca = base["image"]["SCA"] self.pointing = get_pointing(self.base, self.visit, self.sca) def get_sky_image(self): - bounds = galsim.BoundsI(xmin=1, ymin=1, - xmax=4088, ymax=4088) + bounds = galsim.BoundsI(xmin=1, ymin=1, xmax=4088, ymax=4088) self.sky_img = galsim.Image(bounds=bounds, wcs=self.pointing.WCS) - self.pointing.WCS.makeSkyImage(self.sky_img, 0.) + self.pointing.WCS.makeSkyImage(self.sky_img, 0.0) - bound_pad = galsim.BoundsI(xmin=1, ymin=1, - xmax=4096, ymax=4096) + bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) im_pad = galsim.Image(bound_pad) im_pad.array[4:-4, 4:-4] = self.sky_img.array[:, :] - effects_list = list(self.base['image']['add_effects']) - if 'background' not in effects_list: + effects_list = list(self.base["image"]["add_effects"]) + if "background" not in effects_list: return self.sky_img - bkg_idx = effects_list.index('background') + bkg_idx = effects_list.index("background") for i in range(bkg_idx, len(effects_list)): effect_name = effects_list[i] - args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) + args = (self.base["image"]["add_effects"][effect_name], self.base, self.logger, self.rng) effect = getattr(effects, effect_name)(*args) im_pad = effect.apply(image=im_pad) @@ -43,5 +41,5 @@ def get_sky_image(self): return self.sky_img - def save_sky_img(self, outdir='.', sky_img_name='sky_img.fits'): + def save_sky_img(self, outdir=".", sky_img_name="sky_img.fits"): self.sky_img.write(sky_img_name, dir=outdir) diff --git a/roman_imsim/effects/utils.py b/roman_imsim/effects/utils.py index bd7834d1..a350f76a 100644 --- a/roman_imsim/effects/utils.py +++ b/roman_imsim/effects/utils.py @@ -4,24 +4,24 @@ from roman_imsim.obseq import ObSeqDataLoader sca_number_to_file = { - 1 : 'SCA_22066_211227_v001.fits', - 2 : 'SCA_21815_211221_v001.fits', - 3 : 'SCA_21946_211225_v001.fits', - 4 : 'SCA_22073_211229_v001.fits', - 5 : 'SCA_21816_211222_v001.fits', - 6 : 'SCA_20663_211102_v001.fits', - 7 : 'SCA_22069_211228_v001.fits', - 8 : 'SCA_21641_211216_v001.fits', - 9 : 'SCA_21813_211219_v001.fits', - 10 : 'SCA_22078_211230_v001.fits', - 11 : 'SCA_21947_211226_v001.fits', - 12 : 'SCA_22077_211230_v001.fits', - 13 : 'SCA_22067_211227_v001.fits', - 14 : 'SCA_21814_211220_v001.fits', - 15 : 'SCA_21645_211228_v001.fits', - 16 : 'SCA_21643_211218_v001.fits', - 17 : 'SCA_21319_211211_v001.fits', - 18 : 'SCA_20833_211116_v001.fits', + 1: "SCA_22066_211227_v001.fits", + 2: "SCA_21815_211221_v001.fits", + 3: "SCA_21946_211225_v001.fits", + 4: "SCA_22073_211229_v001.fits", + 5: "SCA_21816_211222_v001.fits", + 6: "SCA_20663_211102_v001.fits", + 7: "SCA_22069_211228_v001.fits", + 8: "SCA_21641_211216_v001.fits", + 9: "SCA_21813_211219_v001.fits", + 10: "SCA_22078_211230_v001.fits", + 11: "SCA_21947_211226_v001.fits", + 12: "SCA_22077_211230_v001.fits", + 13: "SCA_22067_211227_v001.fits", + 14: "SCA_21814_211220_v001.fits", + 15: "SCA_21645_211228_v001.fits", + 16: "SCA_21643_211218_v001.fits", + 17: "SCA_21319_211211_v001.fits", + 18: "SCA_20833_211116_v001.fits", } @@ -33,27 +33,27 @@ class get_pointing(object): def __init__(self, params, visit, SCA): self.params = params - file_name = params['input']['obseq_data']['file_name'] + file_name = params["input"]["obseq_data"]["file_name"] obseq_data = ObSeqDataLoader(file_name, visit, SCA, logger=None) - self.filter = obseq_data.ob['filter'] - self.sca = obseq_data.ob['sca'] - self.visit = obseq_data.ob['visit'] - self.date = obseq_data.ob['date'] - self.exptime = obseq_data.ob['exptime'] + self.filter = obseq_data.ob["filter"] + self.sca = obseq_data.ob["sca"] + self.visit = obseq_data.ob["visit"] + self.date = obseq_data.ob["date"] + self.exptime = obseq_data.ob["exptime"] self.bpass = roman.getBandpasses()[self.filter] - self.WCS = roman.getWCS(world_pos=galsim.CelestialCoord(ra=obseq_data.ob['ra'], - dec=obseq_data.ob['dec']), - PA=obseq_data.ob['pa'], - date=self.date, - SCAs=self.sca, - PA_is_FPA=True - )[self.sca] - self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix/2), int(roman.n_pix/2))) + self.WCS = roman.getWCS( + world_pos=galsim.CelestialCoord(ra=obseq_data.ob["ra"], dec=obseq_data.ob["dec"]), + PA=obseq_data.ob["pa"], + date=self.date, + SCAs=self.sca, + PA_is_FPA=True, + )[self.sca] + self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix / 2), int(roman.n_pix / 2))) def translate_cvz(orig_radec, field_ra=9.5, field_dec=-44, cvz_ra=61.24, cvz_dec=-48.42): - ra = orig_radec.ra/galsim.degrees-field_ra - dec = orig_radec.dec/galsim.degrees-field_dec - ra += cvz_ra / np.cos(cvz_dec*np.pi/180) + ra = orig_radec.ra / galsim.degrees - field_ra + dec = orig_radec.dec / galsim.degrees - field_dec + ra += cvz_ra / np.cos(cvz_dec * np.pi / 180) dec += cvz_dec - return galsim.CelestialCoord(ra*galsim.degrees, dec*galsim.degrees) + return galsim.CelestialCoord(ra * galsim.degrees, dec * galsim.degrees) diff --git a/roman_imsim/effects/vtpe.py b/roman_imsim/effects/vtpe.py index 49d55a4f..b695f51a 100644 --- a/roman_imsim/effects/vtpe.py +++ b/roman_imsim/effects/vtpe.py @@ -9,10 +9,12 @@ class vtpe(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params['model']) + self.model = getattr(self, self.params["model"]) if self.model is None: - self.logger.warning("%s hasn't been implemented yet, the simple model will be applied for %s"%( - str(self.params['model']), str(self.__class__.__name__))) + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) self.model = self.simple_model def simple_model(self, image): @@ -43,17 +45,17 @@ def lab_model(self, image): # expand 512x512 arrays to 4096x4096 t = np.zeros((4096, 512)) for row in range(t.shape[0]): - t[row, row//8] = 1 - a_vtpe = t.dot(self.df['VTPE'][0, :, :][0]).dot(t.T) + t[row, row // 8] = 1 + a_vtpe = t.dot(self.df["VTPE"][0, :, :][0]).dot(t.T) # NaN check if np.isnan(a_vtpe).any(): self.logger.warning("vtpe skipped due to NaN in file") return image - b_vtpe = t.dot(self.df['VTPE'][1, :, :][0]).dot(t.T) - dQ0 = t.dot(self.df['VTPE'][2, :, :][0]).dot(t.T) + b_vtpe = t.dot(self.df["VTPE"][1, :, :][0]).dot(t.T) + dQ0 = t.dot(self.df["VTPE"][2, :, :][0]).dot(t.T) dQ = image.array - np.roll(image.array, 1, axis=0) dQ[0, :] *= 0 - image.array[:, :] += dQ * (a_vtpe + b_vtpe * np.log(1. + np.abs(dQ)/dQ0)) + image.array[:, :] += dQ * (a_vtpe + b_vtpe * np.log(1.0 + np.abs(dQ) / dQ0)) return image From f727aac062ccace7af0c31d68248411e22dd34bd Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 29 Jul 2025 18:30:29 -0400 Subject: [PATCH 19/26] black reformated --- roman_imsim/sca.py | 52 +++++++++++++++++++++------------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 7acc5648..24e7cdd7 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -54,26 +54,23 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): ] req = {"SCA": int, "filter": str, "mjd": float, "exptime": float} opt = { - 'draw_method' : str, - 'stray_light' : bool, - 'thermal_background' : bool, - 'reciprocity_failure' : bool, - 'dark_current' : bool, - 'ipc' : bool, - 'read_noise' : bool, - 'sky_subtract' : bool, - 'ignore_noise' : bool, - 'sca_filepath' : str, - 'dither_from_file': str, - 'save_diff': bool, - - 'quantum_efficiency': str, - # 'brighter_fatter': str, - # 'nonlinearity' : str, - 'brighter_fatter': bool, - 'nonlinearity' : bool, - 'add_effects' : dict, + "draw_method": str, + # "stray_light": bool, + # "thermal_background": bool, + # "reciprocity_failure": bool, + # "dark_current": bool, + # "nonlinearity": bool, + # "ipc": bool, + # "read_noise": bool, + "add_effects": dict, + "sca_filepath": str, + "sky_subtract": bool, + "ignore_noise": bool, + "save_diff": bool, } + + logger.warning("opt dict = %s" % (str(opt))) + params = galsim.config.GetAllParams(config, base, req=req, opt=opt, ignore=ignore + extra_ignore)[0] self.sca = params["SCA"] @@ -113,9 +110,9 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): # config['wcs'] = wcs self.rng = galsim.config.GetRNG(config, base) - self.visit = int(base['input']['obseq_data']['visit']) + self.visit = int(base["input"]["obseq_data"]["visit"]) - self.sca_filepath = params.get('sca_filepath', None) + self.sca_filepath = params.get("sca_filepath", None) # If user hasn't overridden the bandpass to use, get the standard one. if "bandpass" not in config: @@ -245,19 +242,18 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) if self.ignore_noise: return - base['current_noise_image'] = base['current_image'] + base["current_noise_image"] = base["current_image"] # rng = galsim.config.GetRNG(config, base) - logger.info('image %d: Start RomanSCA detector effects', base.get('image_num', 0)) + logger.info("image %d: Start RomanSCA detector effects", base.get("image_num", 0)) # create padded image - bound_pad = galsim.BoundsI(xmin=1, ymin=1, - xmax=4096, ymax=4096) + bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) im_pad = galsim.Image(bound_pad) im_pad.array[4:-4, 4:-4] = image.array[:, :] - effects_list = self.base['image']['add_effects'].keys() + effects_list = self.base["image"]["add_effects"].keys() for effect_name in effects_list: - args = (self.base['image']['add_effects'][effect_name], self.base, self.logger, self.rng) + args = (self.base["image"]["add_effects"][effect_name], self.base, self.logger, self.rng) effect = getattr(roman_effects, effect_name)(*args) im_pad = effect.apply(image=im_pad) @@ -270,7 +266,7 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) sky = roman_effects.setup_sky(self.base, self.logger, self.rng) sky_image = sky.get_sky_image() image -= sky_image - sky.save_sky_img(outdir=self.base['output']['dir']) + sky.save_sky_img(outdir=self.base["output"]["dir"]) # Register this as a valid type From 72555822539af0d39a7d4c3de6855a2b421e1b32 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 30 Jul 2025 19:39:31 -0400 Subject: [PATCH 20/26] black formatted --- roman_imsim/sca.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 24e7cdd7..dd2e4584 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -82,18 +82,18 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.ignore_noise = params.get("ignore_noise", False) # self.exptime = params.get('exptime', roman.exptime) # Default is roman standard exposure time. - self.stray_light = params.get('stray_light', False) - self.thermal_background = params.get('thermal_background', False) - self.reciprocity_failure = params.get('reciprocity_failure', False) - self.dark_current = params.get('dark_current', False) - self.ipc = params.get('ipc', False) - self.read_noise = params.get('read_noise', False) - self.sky_subtract = params.get('sky_subtract', False) + self.stray_light = params.get("stray_light", False) + self.thermal_background = params.get("thermal_background", False) + self.reciprocity_failure = params.get("reciprocity_failure", False) + self.dark_current = params.get("dark_current", False) + self.ipc = params.get("ipc", False) + self.read_noise = params.get("read_noise", False) + self.sky_subtract = params.get("sky_subtract", False) # [TODO]TEST - self.qe = params.get('quantum_efficiency', None) - self.bfe = params.get('brighter_fatter', False) - self.nonlinearity = params.get('nonlinearity', False) + self.qe = params.get("quantum_efficiency", None) + self.bfe = params.get("brighter_fatter", False) + self.nonlinearity = params.get("nonlinearity", False) # If draw_method isn't in image field, it may be in stamp. Check. self.draw_method = params.get("draw_method", base.get("stamp", {}).get("draw_method", "auto")) From a76577380d25348da12e67ef2ff2815b6dac17ab Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Fri, 1 Aug 2025 15:17:40 -0400 Subject: [PATCH 21/26] * renaming the classes using CamelCase * removing some commented-out codes * move setup_sky to RomanEffects file. --- config/was.yaml | 34 ++++++------- .../background.py => Effects/Background.py} | 6 +-- .../{effects/bias.py => Effects/Bias.py} | 6 +-- .../BrighterFatter.py} | 8 +-- .../DarkCurrent.py} | 6 +-- .../dead_pix.py => Effects/DeadPixel.py} | 6 +-- .../{effects/gain.py => Effects/Gain.py} | 6 +-- .../interpix_cap.py => Effects/IPC.py} | 6 +-- .../Nonlinearity.py} | 6 +-- .../persistence.py => Effects/Persistence.py} | 10 ++-- .../QuantumEfficiency.py} | 6 +-- .../read_noise.py => Effects/ReadNoise.py} | 6 +-- .../ReciprocityFailure.py} | 6 +-- .../RomanEffects.py} | 49 +++++++++++++++++-- .../saturate.py => Effects/Saturation.py} | 6 +-- .../{effects/vtpe.py => Effects/VTPE.py} | 6 +-- roman_imsim/Effects/__init__.py | 15 ++++++ roman_imsim/{effects => Effects}/utils.py | 0 roman_imsim/effects/__init__.py | 16 ------ roman_imsim/effects/setup_sky.py | 45 ----------------- roman_imsim/sca.py | 29 ++--------- 21 files changed, 123 insertions(+), 155 deletions(-) rename roman_imsim/{effects/background.py => Effects/Background.py} (94%) rename roman_imsim/{effects/bias.py => Effects/Bias.py} (89%) rename roman_imsim/{effects/brighter_fatter.py => Effects/BrighterFatter.py} (98%) rename roman_imsim/{effects/dark_current.py => Effects/DarkCurrent.py} (92%) rename roman_imsim/{effects/dead_pix.py => Effects/DeadPixel.py} (89%) rename roman_imsim/{effects/gain.py => Effects/Gain.py} (91%) rename roman_imsim/{effects/interpix_cap.py => Effects/IPC.py} (95%) rename roman_imsim/{effects/nonlinearity.py => Effects/Nonlinearity.py} (92%) rename roman_imsim/{effects/persistence.py => Effects/Persistence.py} (95%) rename roman_imsim/{effects/quantum_efficiency.py => Effects/QuantumEfficiency.py} (86%) rename roman_imsim/{effects/read_noise.py => Effects/ReadNoise.py} (91%) rename roman_imsim/{effects/recip_failure.py => Effects/ReciprocityFailure.py} (87%) rename roman_imsim/{effects/roman_effects.py => Effects/RomanEffects.py} (61%) rename roman_imsim/{effects/saturate.py => Effects/Saturation.py} (92%) rename roman_imsim/{effects/vtpe.py => Effects/VTPE.py} (94%) create mode 100644 roman_imsim/Effects/__init__.py rename roman_imsim/{effects => Effects}/utils.py (100%) delete mode 100644 roman_imsim/effects/__init__.py delete mode 100644 roman_imsim/effects/setup_sky.py diff --git a/config/was.yaml b/config/was.yaml index 879d90dd..589072d9 100644 --- a/config/was.yaml +++ b/config/was.yaml @@ -89,39 +89,38 @@ image: # sky_subtract: False add_effects: - background: + Background: model: simple_model thermal_background: True stray_light: True - quantum_efficiency: + QuantumEfficiency: model: lab_model - brighter_fatter: + BrighterFatter: model: lab_model - persistence: + Persistence: model: lab_model - # reciprocity_failure: + # ReciprocityFailure: # model: simple_model - dark_current: + DarkCurrent: model: lab_model - saturate: + Saturation: model: lab_model - nonlinearity: + Nonlinearity: model: lab_model - interpix_cap: + IPC: model: lab_model - dead_pix: + DeadPixel: model: lab_model - vtpe: + VTPE: model: lab_model - read_noise: + ReadNoise: model: lab_model - gain: + Gain: model: lab_model - bias: + Bias: model: lab_model sky_subtract: False - # dither_from_file: /hpc/home/yf194/Work/final_sims_2023/Roman_WAS_obseq_11_1_23_ditherlist.txt sca_filepath: /hpc/group/cosmology/phy-lsst/roman_sensors save_diff: False @@ -151,7 +150,6 @@ gal: input: obseq_data: - # file_name: /lus/grand/projects/RomanDESC/final_roman_runfiles/was/Roman_WAS_obseq_11_1_23.fits file_name: /hpc/group/cosmology/OpenUniverse2024/RomanWAS/Roman_WAS_obseq_11_1_23.fits visit: 4906 SCA: '@image.SCA' @@ -159,9 +157,7 @@ input: SCA: '@image.SCA' n_waves: 5 sky_catalog: - # file_name: /lus/grand/projects/RomanDESC/roman_rubin_cats_v1.1.2_faint/skyCatalog.yaml - # file_name: /hpc/group/cosmology/OpenUniverse2024/roman_rubin_cats_v1.1.2_faint/skyCatalog.yaml - file_name: /hpc/home/yf194/Work/projects/roman_imsim/config/skyCatalog.yaml + file_name: /hpc/group/cosmology/OpenUniverse2024/roman_rubin_cats_v1.1.2_faint/skyCatalog.yaml edge_pix: 512 mjd: { type: ObSeqData, field: mjd } exptime: { type: ObSeqData, field: exptime } diff --git a/roman_imsim/effects/background.py b/roman_imsim/Effects/Background.py similarity index 94% rename from roman_imsim/effects/background.py rename to roman_imsim/Effects/Background.py index f940d029..47312db3 100644 --- a/roman_imsim/effects/background.py +++ b/roman_imsim/Effects/Background.py @@ -1,10 +1,10 @@ import os import galsim -from roman_imsim.effects import roman_effects +from . import RomanEffects import galsim.roman as roman -class background(roman_effects): +class Background(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) self.thermal_background = ( @@ -12,7 +12,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): ) self.stray_light = self.params["stray_light"] if "stray_light" in self.params else False - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/bias.py b/roman_imsim/Effects/Bias.py similarity index 89% rename from roman_imsim/effects/bias.py rename to roman_imsim/Effects/Bias.py index eb6a5db9..29e2ec09 100644 --- a/roman_imsim/effects/bias.py +++ b/roman_imsim/Effects/Bias.py @@ -1,14 +1,14 @@ import os import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class bias(roman_effects): +class Bias(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/brighter_fatter.py b/roman_imsim/Effects/BrighterFatter.py similarity index 98% rename from roman_imsim/effects/brighter_fatter.py rename to roman_imsim/Effects/BrighterFatter.py index 9b334a99..295ec960 100644 --- a/roman_imsim/effects/brighter_fatter.py +++ b/roman_imsim/Effects/BrighterFatter.py @@ -1,15 +1,15 @@ import os import numpy as np import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class brighter_fatter(roman_effects): +class BrighterFatter(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" @@ -128,7 +128,7 @@ def lab_model(self, image): # where_sat = np.where(array_pad > saturation_array) # array_pad[ where_sat ] = saturation_array[ where_sat ] # array_pad = array_pad[4:-4,4:-4] - saturate = self.cross_refer("saturate") + saturate = self.cross_refer("Saturation") array_pad = saturate.apply(image=image.copy()).array[4:-4, 4:-4] # img of interest 4088x4088 array_pad = np.pad( array_pad, [(4 + nbfe, 4 + nbfe), (4 + nbfe, 4 + nbfe)], mode="symmetric" diff --git a/roman_imsim/effects/dark_current.py b/roman_imsim/Effects/DarkCurrent.py similarity index 92% rename from roman_imsim/effects/dark_current.py rename to roman_imsim/Effects/DarkCurrent.py index 77032468..a4fc3e94 100644 --- a/roman_imsim/effects/dark_current.py +++ b/roman_imsim/Effects/DarkCurrent.py @@ -2,15 +2,15 @@ import fitsio as fio import galsim import galsim.roman as roman -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class dark_current(roman_effects): +class DarkCurrent(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/dead_pix.py b/roman_imsim/Effects/DeadPixel.py similarity index 89% rename from roman_imsim/effects/dead_pix.py rename to roman_imsim/Effects/DeadPixel.py index 2580fb14..485ecf8d 100644 --- a/roman_imsim/effects/dead_pix.py +++ b/roman_imsim/Effects/DeadPixel.py @@ -1,14 +1,14 @@ import os import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class dead_pix(roman_effects): +class DeadPixel(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/gain.py b/roman_imsim/Effects/Gain.py similarity index 91% rename from roman_imsim/effects/gain.py rename to roman_imsim/Effects/Gain.py index e82fce68..d69da928 100644 --- a/roman_imsim/effects/gain.py +++ b/roman_imsim/Effects/Gain.py @@ -2,15 +2,15 @@ import numpy as np import fitsio as fio import galsim.roman as roman -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class gain(roman_effects): +class Gain(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/interpix_cap.py b/roman_imsim/Effects/IPC.py similarity index 95% rename from roman_imsim/effects/interpix_cap.py rename to roman_imsim/Effects/IPC.py index 4de17a06..adb475e0 100644 --- a/roman_imsim/effects/interpix_cap.py +++ b/roman_imsim/Effects/IPC.py @@ -2,15 +2,15 @@ import numpy as np import fitsio as fio import galsim.roman as roman -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class interpix_cap(roman_effects): +class IPC(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/nonlinearity.py b/roman_imsim/Effects/Nonlinearity.py similarity index 92% rename from roman_imsim/effects/nonlinearity.py rename to roman_imsim/Effects/Nonlinearity.py index 8be3e955..213e4fb6 100644 --- a/roman_imsim/effects/nonlinearity.py +++ b/roman_imsim/Effects/Nonlinearity.py @@ -1,11 +1,11 @@ import os import fitsio as fio import galsim.roman as roman -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class nonlinearity(roman_effects): +class Nonlinearity(RomanEffects): """ Applying a quadratic non-linearity. @@ -20,7 +20,7 @@ class nonlinearity(roman_effects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/persistence.py b/roman_imsim/Effects/Persistence.py similarity index 95% rename from roman_imsim/effects/persistence.py rename to roman_imsim/Effects/Persistence.py index 17616765..13929f82 100644 --- a/roman_imsim/effects/persistence.py +++ b/roman_imsim/Effects/Persistence.py @@ -2,15 +2,15 @@ import numpy as np import fitsio as fio import galsim -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file, get_pointing -class persistence(roman_effects): +class Persistence(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" @@ -43,7 +43,7 @@ def simple_model(self, image): x = galsim.Image(bound_pad) x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] - recip_failure = self.cross_refer("recip_failure") + recip_failure = self.cross_refer("ReciprocityFailure") x = recip_failure.apply(image=x) x.array.clip(0) # remove negative stimulus @@ -96,7 +96,7 @@ def lab_model(self, image): x.array[4:-4, 4:-4] = galsim.Image(fio.FITS(fn)[0].read()).array[:, :] # x = self.qe(x).array[:,:] - qe = self.cross_refer("quantum_efficiency") + qe = self.cross_refer("QuantumEfficiency") x = qe.apply(image=x).array[:, :] x = x.clip(0.1) # remove negative and zero stimulus diff --git a/roman_imsim/effects/quantum_efficiency.py b/roman_imsim/Effects/QuantumEfficiency.py similarity index 86% rename from roman_imsim/effects/quantum_efficiency.py rename to roman_imsim/Effects/QuantumEfficiency.py index 637b8766..3d18894f 100644 --- a/roman_imsim/effects/quantum_efficiency.py +++ b/roman_imsim/Effects/QuantumEfficiency.py @@ -1,14 +1,14 @@ import os import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class quantum_efficiency(roman_effects): +class QuantumEfficiency(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/read_noise.py b/roman_imsim/Effects/ReadNoise.py similarity index 91% rename from roman_imsim/effects/read_noise.py rename to roman_imsim/Effects/ReadNoise.py index 048dd43c..c036f08c 100644 --- a/roman_imsim/effects/read_noise.py +++ b/roman_imsim/Effects/ReadNoise.py @@ -1,14 +1,14 @@ import os import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class read_noise(roman_effects): +class ReadNoise(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/recip_failure.py b/roman_imsim/Effects/ReciprocityFailure.py similarity index 87% rename from roman_imsim/effects/recip_failure.py rename to roman_imsim/Effects/ReciprocityFailure.py index 8bc410d0..54f3ba7f 100644 --- a/roman_imsim/effects/recip_failure.py +++ b/roman_imsim/Effects/ReciprocityFailure.py @@ -1,15 +1,15 @@ -from roman_imsim.effects import roman_effects +from . import RomanEffects import galsim.roman as roman -class recip_failure(roman_effects): +class ReciprocityFailure(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) self.alpha = self.params["alpha"] if "alpha" in self.params else roman.reciprocity_alpha self.base_flux = self.params["base_flux"] if "base_flux" in self.params else 1.0 - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/roman_effects.py b/roman_imsim/Effects/RomanEffects.py similarity index 61% rename from roman_imsim/effects/roman_effects.py rename to roman_imsim/Effects/RomanEffects.py index 1ff96837..d14a6873 100644 --- a/roman_imsim/effects/roman_effects.py +++ b/roman_imsim/Effects/RomanEffects.py @@ -1,10 +1,10 @@ import galsim as galsim import numpy as np from .utils import get_pointing -import roman_imsim.effects as effects +import roman_imsim.Effects as Effects -class roman_effects(object): +class RomanEffects(object): """ Class to simulate non-idealities and noise of roman detector images. """ @@ -45,7 +45,7 @@ def simple_model(self, image): def cross_refer(self, effect_name): if effect_name not in self.base["image"]["add_effects"]: try: - effect = getattr(effects, effect_name)( + effect = getattr(Effects, effect_name)( {"model": "simple_model"}, self.base, self.logger, self.rng ) except Exception as e: @@ -55,7 +55,7 @@ def cross_refer(self, effect_name): else: try: params = self.base["image"]["add_effects"][effect_name] - effect = getattr(effects, effect_name)(params, self.base, self.logger, self.rng) + effect = getattr(Effects, effect_name)(params, self.base, self.logger, self.rng) except Exception as e: self.logger.warning(e) # self.logger.warning("Effect %s is not implemented!"%(effect_name)) @@ -79,3 +79,44 @@ def diff(self, msg, im=None, verbose=True): self.pre = im.copy() im.write("%s_cumul.fits" % msg, dir=self.diff_dir) return + + +class setup_sky(object): + def __init__(self, base, logger, rng, rng_iter=None): + self.base = base + self.logger = logger + self.rng = rng + + self.visit = int(self.base["input"]["obseq_data"]["visit"]) + self.sca = base["image"]["SCA"] + self.pointing = get_pointing(self.base, self.visit, self.sca) + + def get_sky_image(self): + bounds = galsim.BoundsI(xmin=1, ymin=1, xmax=4088, ymax=4088) + self.sky_img = galsim.Image(bounds=bounds, wcs=self.pointing.WCS) + self.pointing.WCS.makeSkyImage(self.sky_img, 0.0) + + bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) + im_pad = galsim.Image(bound_pad) + im_pad.array[4:-4, 4:-4] = self.sky_img.array[:, :] + + effects_list = list(self.base["image"]["add_effects"]) + if "Background" not in effects_list: + return self.sky_img + + bkg_idx = effects_list.index("Background") + for i in range(bkg_idx, len(effects_list)): + effect_name = effects_list[i] + args = (self.base["image"]["add_effects"][effect_name], self.base, self.logger, self.rng) + effect = getattr(Effects, effect_name)(*args) + im_pad = effect.apply(image=im_pad) + + im_pad.quantize() + # output 4088x4088 img in uint16 + self.sky_img = im_pad.array[4:-4, 4:-4] + self.sky_img = galsim.Image(self.sky_img, dtype=np.uint16) + + return self.sky_img + + def save_sky_img(self, outdir=".", sky_img_name="sky_img.fits"): + self.sky_img.write(sky_img_name, dir=outdir) diff --git a/roman_imsim/effects/saturate.py b/roman_imsim/Effects/Saturation.py similarity index 92% rename from roman_imsim/effects/saturate.py rename to roman_imsim/Effects/Saturation.py index 01f29679..6bfba970 100644 --- a/roman_imsim/effects/saturate.py +++ b/roman_imsim/Effects/Saturation.py @@ -1,15 +1,15 @@ import os import numpy as np import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class saturate(roman_effects): +class Saturation(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/effects/vtpe.py b/roman_imsim/Effects/VTPE.py similarity index 94% rename from roman_imsim/effects/vtpe.py rename to roman_imsim/Effects/VTPE.py index b695f51a..019c7176 100644 --- a/roman_imsim/effects/vtpe.py +++ b/roman_imsim/Effects/VTPE.py @@ -1,15 +1,15 @@ import os import numpy as np import fitsio as fio -from roman_imsim.effects import roman_effects +from . import RomanEffects from .utils import sca_number_to_file -class vtpe(roman_effects): +class VTPE(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"]) + self.model = getattr(self, self.params["model"], None) if self.model is None: self.logger.warning( "%s hasn't been implemented yet, the simple model will be applied for %s" diff --git a/roman_imsim/Effects/__init__.py b/roman_imsim/Effects/__init__.py new file mode 100644 index 00000000..4c96976a --- /dev/null +++ b/roman_imsim/Effects/__init__.py @@ -0,0 +1,15 @@ +from .RomanEffects import RomanEffects, setup_sky +from .BrighterFatter import BrighterFatter +from .Nonlinearity import Nonlinearity +from .Background import Background +from .QuantumEfficiency import QuantumEfficiency +from .Persistence import Persistence +from .ReciprocityFailure import ReciprocityFailure +from .DarkCurrent import DarkCurrent +from .Saturation import Saturation +from .IPC import IPC +from .DeadPixel import DeadPixel +from .VTPE import VTPE +from .ReadNoise import ReadNoise +from .Gain import Gain +from .Bias import Bias diff --git a/roman_imsim/effects/utils.py b/roman_imsim/Effects/utils.py similarity index 100% rename from roman_imsim/effects/utils.py rename to roman_imsim/Effects/utils.py diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py deleted file mode 100644 index 3ff4da12..00000000 --- a/roman_imsim/effects/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .roman_effects import roman_effects -from .brighter_fatter import brighter_fatter -from .nonlinearity import nonlinearity -from .background import background -from .quantum_efficiency import quantum_efficiency -from .persistence import persistence -from .recip_failure import recip_failure -from .dark_current import dark_current -from .saturate import saturate -from .interpix_cap import interpix_cap -from .dead_pix import dead_pix -from .vtpe import vtpe -from .read_noise import read_noise -from .gain import gain -from .bias import bias -from .setup_sky import setup_sky diff --git a/roman_imsim/effects/setup_sky.py b/roman_imsim/effects/setup_sky.py deleted file mode 100644 index 1da94895..00000000 --- a/roman_imsim/effects/setup_sky.py +++ /dev/null @@ -1,45 +0,0 @@ -import numpy as np -import galsim -import roman_imsim.effects as effects -from .utils import get_pointing - - -class setup_sky(object): - def __init__(self, base, logger, rng, rng_iter=None): - self.base = base - self.logger = logger - self.rng = rng - - self.visit = int(self.base["input"]["obseq_data"]["visit"]) - self.sca = base["image"]["SCA"] - self.pointing = get_pointing(self.base, self.visit, self.sca) - - def get_sky_image(self): - bounds = galsim.BoundsI(xmin=1, ymin=1, xmax=4088, ymax=4088) - self.sky_img = galsim.Image(bounds=bounds, wcs=self.pointing.WCS) - self.pointing.WCS.makeSkyImage(self.sky_img, 0.0) - - bound_pad = galsim.BoundsI(xmin=1, ymin=1, xmax=4096, ymax=4096) - im_pad = galsim.Image(bound_pad) - im_pad.array[4:-4, 4:-4] = self.sky_img.array[:, :] - - effects_list = list(self.base["image"]["add_effects"]) - if "background" not in effects_list: - return self.sky_img - - bkg_idx = effects_list.index("background") - for i in range(bkg_idx, len(effects_list)): - effect_name = effects_list[i] - args = (self.base["image"]["add_effects"][effect_name], self.base, self.logger, self.rng) - effect = getattr(effects, effect_name)(*args) - im_pad = effect.apply(image=im_pad) - - im_pad.quantize() - # output 4088x4088 img in uint16 - self.sky_img = im_pad.array[4:-4, 4:-4] - self.sky_img = galsim.Image(self.sky_img, dtype=np.uint16) - - return self.sky_img - - def save_sky_img(self, outdir=".", sky_img_name="sky_img.fits"): - self.sky_img.write(sky_img_name, dir=outdir) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index dd2e4584..5dea767b 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -8,7 +8,7 @@ from galsim.image import Image -import roman_imsim.effects as roman_effects +import roman_imsim.Effects as RomanEffects class RomanSCAImageBuilder(ScatteredImageBuilder): @@ -55,13 +55,6 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): req = {"SCA": int, "filter": str, "mjd": float, "exptime": float} opt = { "draw_method": str, - # "stray_light": bool, - # "thermal_background": bool, - # "reciprocity_failure": bool, - # "dark_current": bool, - # "nonlinearity": bool, - # "ipc": bool, - # "read_noise": bool, "add_effects": dict, "sca_filepath": str, "sky_subtract": bool, @@ -80,21 +73,8 @@ def setup(self, config, base, image_num, obj_num, ignore, logger): self.exptime = params["exptime"] self.ignore_noise = params.get("ignore_noise", False) - - # self.exptime = params.get('exptime', roman.exptime) # Default is roman standard exposure time. - self.stray_light = params.get("stray_light", False) - self.thermal_background = params.get("thermal_background", False) - self.reciprocity_failure = params.get("reciprocity_failure", False) - self.dark_current = params.get("dark_current", False) - self.ipc = params.get("ipc", False) - self.read_noise = params.get("read_noise", False) self.sky_subtract = params.get("sky_subtract", False) - # [TODO]TEST - self.qe = params.get("quantum_efficiency", None) - self.bfe = params.get("brighter_fatter", False) - self.nonlinearity = params.get("nonlinearity", False) - # If draw_method isn't in image field, it may be in stamp. Check. self.draw_method = params.get("draw_method", base.get("stamp", {}).get("draw_method", "auto")) @@ -217,9 +197,6 @@ def buildImage(self, config, base, image_num, obj_num, logger): full_image[bounds] += stamps[k][bounds] stamps = None - # # [TODO] - # break - # # Bring the image so far up to a flat noise variance # current_var = FlattenNoiseVariance( # base, full_image, stamps, current_vars, logger) @@ -254,7 +231,7 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) effects_list = self.base["image"]["add_effects"].keys() for effect_name in effects_list: args = (self.base["image"]["add_effects"][effect_name], self.base, self.logger, self.rng) - effect = getattr(roman_effects, effect_name)(*args) + effect = getattr(RomanEffects, effect_name)(*args) im_pad = effect.apply(image=im_pad) im_pad.quantize() @@ -263,7 +240,7 @@ def addNoise(self, image, config, base, image_num, obj_num, current_var, logger) if self.sky_subtract: logger.debug("Subtracting sky image") - sky = roman_effects.setup_sky(self.base, self.logger, self.rng) + sky = RomanEffects.setup_sky(self.base, self.logger, self.rng) sky_image = sky.get_sky_image() image -= sky_image sky.save_sky_img(outdir=self.base["output"]["dir"]) From 7c6bf21c041db25f9824bd83c0ea9a7290c2df1a Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Fri, 1 Aug 2025 16:09:06 -0400 Subject: [PATCH 22/26] move model validity checking into base (RomanEffects) class --- roman_imsim/Effects/Background.py | 8 +------- roman_imsim/Effects/Bias.py | 8 +------- roman_imsim/Effects/BrighterFatter.py | 8 +------- roman_imsim/Effects/DarkCurrent.py | 8 +------- roman_imsim/Effects/DeadPixel.py | 8 +------- roman_imsim/Effects/Gain.py | 8 +------- roman_imsim/Effects/IPC.py | 8 +------- roman_imsim/Effects/Nonlinearity.py | 8 +------- roman_imsim/Effects/Persistence.py | 8 +------- roman_imsim/Effects/QuantumEfficiency.py | 8 +------- roman_imsim/Effects/ReadNoise.py | 8 +------- roman_imsim/Effects/ReciprocityFailure.py | 8 +------- roman_imsim/Effects/RomanEffects.py | 9 +++++++++ roman_imsim/Effects/Saturation.py | 8 +------- roman_imsim/Effects/VTPE.py | 8 +------- 15 files changed, 23 insertions(+), 98 deletions(-) diff --git a/roman_imsim/Effects/Background.py b/roman_imsim/Effects/Background.py index 47312db3..b16b7374 100644 --- a/roman_imsim/Effects/Background.py +++ b/roman_imsim/Effects/Background.py @@ -12,13 +12,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): ) self.stray_light = self.params["stray_light"] if "stray_light" in self.params else False - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): if self.save_diff: diff --git a/roman_imsim/Effects/Bias.py b/roman_imsim/Effects/Bias.py index 29e2ec09..54a3a574 100644 --- a/roman_imsim/Effects/Bias.py +++ b/roman_imsim/Effects/Bias.py @@ -8,13 +8,7 @@ class Bias(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("No bias will be applied.") diff --git a/roman_imsim/Effects/BrighterFatter.py b/roman_imsim/Effects/BrighterFatter.py index 295ec960..bad18085 100644 --- a/roman_imsim/Effects/BrighterFatter.py +++ b/roman_imsim/Effects/BrighterFatter.py @@ -9,13 +9,7 @@ class BrighterFatter(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("No bfe effect will be applied.") diff --git a/roman_imsim/Effects/DarkCurrent.py b/roman_imsim/Effects/DarkCurrent.py index a4fc3e94..252ffb43 100644 --- a/roman_imsim/Effects/DarkCurrent.py +++ b/roman_imsim/Effects/DarkCurrent.py @@ -10,13 +10,7 @@ class DarkCurrent(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("Simple model will be applied for dark current.") diff --git a/roman_imsim/Effects/DeadPixel.py b/roman_imsim/Effects/DeadPixel.py index 485ecf8d..320bddf7 100644 --- a/roman_imsim/Effects/DeadPixel.py +++ b/roman_imsim/Effects/DeadPixel.py @@ -8,13 +8,7 @@ class DeadPixel(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("No dead pixel will be applied.") diff --git a/roman_imsim/Effects/Gain.py b/roman_imsim/Effects/Gain.py index d69da928..c90cb55f 100644 --- a/roman_imsim/Effects/Gain.py +++ b/roman_imsim/Effects/Gain.py @@ -10,13 +10,7 @@ class Gain(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("Simple model will be applied for gain.") diff --git a/roman_imsim/Effects/IPC.py b/roman_imsim/Effects/IPC.py index adb475e0..ebd13c70 100644 --- a/roman_imsim/Effects/IPC.py +++ b/roman_imsim/Effects/IPC.py @@ -10,13 +10,7 @@ class IPC(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("Simple model will be applied for IPC effect.") diff --git a/roman_imsim/Effects/Nonlinearity.py b/roman_imsim/Effects/Nonlinearity.py index 213e4fb6..f2f43862 100644 --- a/roman_imsim/Effects/Nonlinearity.py +++ b/roman_imsim/Effects/Nonlinearity.py @@ -20,13 +20,7 @@ class Nonlinearity(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.info("Galsim.roman.NLfunc will be applied for simulating non-linearity effect.") diff --git a/roman_imsim/Effects/Persistence.py b/roman_imsim/Effects/Persistence.py index 13929f82..d814be15 100644 --- a/roman_imsim/Effects/Persistence.py +++ b/roman_imsim/Effects/Persistence.py @@ -10,13 +10,7 @@ class Persistence(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() p_list = np.array([get_pointing(self.base, i, self.sca) for i in range(self.visit - 10, self.visit)]) dt_list = np.array([(self.pointing.date - p.date).total_seconds() for p in p_list]) diff --git a/roman_imsim/Effects/QuantumEfficiency.py b/roman_imsim/Effects/QuantumEfficiency.py index 3d18894f..7c5951ac 100644 --- a/roman_imsim/Effects/QuantumEfficiency.py +++ b/roman_imsim/Effects/QuantumEfficiency.py @@ -8,13 +8,7 @@ class QuantumEfficiency(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def lab_model(self, image): if self.sca_filepath is None: diff --git a/roman_imsim/Effects/ReadNoise.py b/roman_imsim/Effects/ReadNoise.py index c036f08c..d001a5ee 100644 --- a/roman_imsim/Effects/ReadNoise.py +++ b/roman_imsim/Effects/ReadNoise.py @@ -8,13 +8,7 @@ class ReadNoise(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("Simple model will be applied for read noise.") diff --git a/roman_imsim/Effects/ReciprocityFailure.py b/roman_imsim/Effects/ReciprocityFailure.py index 54f3ba7f..ef577045 100644 --- a/roman_imsim/Effects/ReciprocityFailure.py +++ b/roman_imsim/Effects/ReciprocityFailure.py @@ -9,13 +9,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): self.alpha = self.params["alpha"] if "alpha" in self.params else roman.reciprocity_alpha self.base_flux = self.params["base_flux"] if "base_flux" in self.params else 1.0 - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): # Add reciprocity effect diff --git a/roman_imsim/Effects/RomanEffects.py b/roman_imsim/Effects/RomanEffects.py index d14a6873..378ed4b7 100644 --- a/roman_imsim/Effects/RomanEffects.py +++ b/roman_imsim/Effects/RomanEffects.py @@ -38,6 +38,15 @@ def __init__(self, params, base, logger, rng, rng_iter=None): else: self.diff_dir = self.base["output"]["dir"] + def is_model_valid(self): + self.model = getattr(self, self.params["model"], None) + if self.model is None: + self.logger.warning( + "%s hasn't been implemented yet, the simple model will be applied for %s" + % (str(self.params["model"]), str(self.__class__.__name__)) + ) + self.model = self.simple_model + def simple_model(self, image): self.logger.info("Applying the default model...") return image diff --git a/roman_imsim/Effects/Saturation.py b/roman_imsim/Effects/Saturation.py index 6bfba970..b2d1ef4d 100644 --- a/roman_imsim/Effects/Saturation.py +++ b/roman_imsim/Effects/Saturation.py @@ -9,13 +9,7 @@ class Saturation(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() self.saturation_level = ( self.params["saturation_level"] if "saturation_level" in self.params else 100000 diff --git a/roman_imsim/Effects/VTPE.py b/roman_imsim/Effects/VTPE.py index 019c7176..92e1e1c8 100644 --- a/roman_imsim/Effects/VTPE.py +++ b/roman_imsim/Effects/VTPE.py @@ -9,13 +9,7 @@ class VTPE(RomanEffects): def __init__(self, params, base, logger, rng, rng_iter=None): super().__init__(params, base, logger, rng, rng_iter) - self.model = getattr(self, self.params["model"], None) - if self.model is None: - self.logger.warning( - "%s hasn't been implemented yet, the simple model will be applied for %s" - % (str(self.params["model"]), str(self.__class__.__name__)) - ) - self.model = self.simple_model + self.is_model_valid() def simple_model(self, image): self.logger.warning("No vertical trailing pixel effect will be applied.") From 24a84ed917edfaa1396459f1c1940483a4f11b50 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 26 Aug 2025 18:20:05 -0400 Subject: [PATCH 23/26] renamed the files for effects --- roman_imsim/Effects/__init__.py | 15 --------------- roman_imsim/effects/__init__.py | 15 +++++++++++++++ .../Background.py => effects/background.py} | 0 roman_imsim/{Effects/Bias.py => effects/bias.py} | 0 .../brighter_fatter.py} | 0 .../DarkCurrent.py => effects/dark_current.py} | 0 .../DeadPixel.py => effects/dead_pixel.py} | 0 roman_imsim/{Effects/Gain.py => effects/gain.py} | 0 roman_imsim/{Effects/IPC.py => effects/ipc.py} | 0 .../Nonlinearity.py => effects/nonlinearity.py} | 0 .../Persistence.py => effects/persistence.py} | 0 .../quantum_efficiency.py} | 0 .../ReadNoise.py => effects/read_noise.py} | 0 .../reciprocity_failure.py} | 0 .../RomanEffects.py => effects/roman_effects.py} | 2 +- .../Saturation.py => effects/saturation.py} | 0 roman_imsim/{Effects => effects}/utils.py | 0 roman_imsim/{Effects/VTPE.py => effects/vtpe.py} | 0 18 files changed, 16 insertions(+), 16 deletions(-) delete mode 100644 roman_imsim/Effects/__init__.py create mode 100644 roman_imsim/effects/__init__.py rename roman_imsim/{Effects/Background.py => effects/background.py} (100%) rename roman_imsim/{Effects/Bias.py => effects/bias.py} (100%) rename roman_imsim/{Effects/BrighterFatter.py => effects/brighter_fatter.py} (100%) rename roman_imsim/{Effects/DarkCurrent.py => effects/dark_current.py} (100%) rename roman_imsim/{Effects/DeadPixel.py => effects/dead_pixel.py} (100%) rename roman_imsim/{Effects/Gain.py => effects/gain.py} (100%) rename roman_imsim/{Effects/IPC.py => effects/ipc.py} (100%) rename roman_imsim/{Effects/Nonlinearity.py => effects/nonlinearity.py} (100%) rename roman_imsim/{Effects/Persistence.py => effects/persistence.py} (100%) rename roman_imsim/{Effects/QuantumEfficiency.py => effects/quantum_efficiency.py} (100%) rename roman_imsim/{Effects/ReadNoise.py => effects/read_noise.py} (100%) rename roman_imsim/{Effects/ReciprocityFailure.py => effects/reciprocity_failure.py} (100%) rename roman_imsim/{Effects/RomanEffects.py => effects/roman_effects.py} (99%) rename roman_imsim/{Effects/Saturation.py => effects/saturation.py} (100%) rename roman_imsim/{Effects => effects}/utils.py (100%) rename roman_imsim/{Effects/VTPE.py => effects/vtpe.py} (100%) diff --git a/roman_imsim/Effects/__init__.py b/roman_imsim/Effects/__init__.py deleted file mode 100644 index 4c96976a..00000000 --- a/roman_imsim/Effects/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .RomanEffects import RomanEffects, setup_sky -from .BrighterFatter import BrighterFatter -from .Nonlinearity import Nonlinearity -from .Background import Background -from .QuantumEfficiency import QuantumEfficiency -from .Persistence import Persistence -from .ReciprocityFailure import ReciprocityFailure -from .DarkCurrent import DarkCurrent -from .Saturation import Saturation -from .IPC import IPC -from .DeadPixel import DeadPixel -from .VTPE import VTPE -from .ReadNoise import ReadNoise -from .Gain import Gain -from .Bias import Bias diff --git a/roman_imsim/effects/__init__.py b/roman_imsim/effects/__init__.py new file mode 100644 index 00000000..7a1faf33 --- /dev/null +++ b/roman_imsim/effects/__init__.py @@ -0,0 +1,15 @@ +from .roman_effects import RomanEffects, setup_sky +from .brighter_fatter import BrighterFatter +from .nonlinearity import Nonlinearity +from .background import Background +from .quantum_efficiency import QuantumEfficiency +from .persistence import Persistence +from .reciprocity_failure import ReciprocityFailure +from .dark_current import DarkCurrent +from .saturation import Saturation +from .ipc import IPC +from .dead_pixel import DeadPixel +from .vtpe import VTPE +from .read_noise import ReadNoise +from .gain import Gain +from .bias import Bias diff --git a/roman_imsim/Effects/Background.py b/roman_imsim/effects/background.py similarity index 100% rename from roman_imsim/Effects/Background.py rename to roman_imsim/effects/background.py diff --git a/roman_imsim/Effects/Bias.py b/roman_imsim/effects/bias.py similarity index 100% rename from roman_imsim/Effects/Bias.py rename to roman_imsim/effects/bias.py diff --git a/roman_imsim/Effects/BrighterFatter.py b/roman_imsim/effects/brighter_fatter.py similarity index 100% rename from roman_imsim/Effects/BrighterFatter.py rename to roman_imsim/effects/brighter_fatter.py diff --git a/roman_imsim/Effects/DarkCurrent.py b/roman_imsim/effects/dark_current.py similarity index 100% rename from roman_imsim/Effects/DarkCurrent.py rename to roman_imsim/effects/dark_current.py diff --git a/roman_imsim/Effects/DeadPixel.py b/roman_imsim/effects/dead_pixel.py similarity index 100% rename from roman_imsim/Effects/DeadPixel.py rename to roman_imsim/effects/dead_pixel.py diff --git a/roman_imsim/Effects/Gain.py b/roman_imsim/effects/gain.py similarity index 100% rename from roman_imsim/Effects/Gain.py rename to roman_imsim/effects/gain.py diff --git a/roman_imsim/Effects/IPC.py b/roman_imsim/effects/ipc.py similarity index 100% rename from roman_imsim/Effects/IPC.py rename to roman_imsim/effects/ipc.py diff --git a/roman_imsim/Effects/Nonlinearity.py b/roman_imsim/effects/nonlinearity.py similarity index 100% rename from roman_imsim/Effects/Nonlinearity.py rename to roman_imsim/effects/nonlinearity.py diff --git a/roman_imsim/Effects/Persistence.py b/roman_imsim/effects/persistence.py similarity index 100% rename from roman_imsim/Effects/Persistence.py rename to roman_imsim/effects/persistence.py diff --git a/roman_imsim/Effects/QuantumEfficiency.py b/roman_imsim/effects/quantum_efficiency.py similarity index 100% rename from roman_imsim/Effects/QuantumEfficiency.py rename to roman_imsim/effects/quantum_efficiency.py diff --git a/roman_imsim/Effects/ReadNoise.py b/roman_imsim/effects/read_noise.py similarity index 100% rename from roman_imsim/Effects/ReadNoise.py rename to roman_imsim/effects/read_noise.py diff --git a/roman_imsim/Effects/ReciprocityFailure.py b/roman_imsim/effects/reciprocity_failure.py similarity index 100% rename from roman_imsim/Effects/ReciprocityFailure.py rename to roman_imsim/effects/reciprocity_failure.py diff --git a/roman_imsim/Effects/RomanEffects.py b/roman_imsim/effects/roman_effects.py similarity index 99% rename from roman_imsim/Effects/RomanEffects.py rename to roman_imsim/effects/roman_effects.py index 378ed4b7..94c51037 100644 --- a/roman_imsim/Effects/RomanEffects.py +++ b/roman_imsim/effects/roman_effects.py @@ -1,7 +1,7 @@ import galsim as galsim import numpy as np from .utils import get_pointing -import roman_imsim.Effects as Effects +import roman_imsim.effects as Effects class RomanEffects(object): diff --git a/roman_imsim/Effects/Saturation.py b/roman_imsim/effects/saturation.py similarity index 100% rename from roman_imsim/Effects/Saturation.py rename to roman_imsim/effects/saturation.py diff --git a/roman_imsim/Effects/utils.py b/roman_imsim/effects/utils.py similarity index 100% rename from roman_imsim/Effects/utils.py rename to roman_imsim/effects/utils.py diff --git a/roman_imsim/Effects/VTPE.py b/roman_imsim/effects/vtpe.py similarity index 100% rename from roman_imsim/Effects/VTPE.py rename to roman_imsim/effects/vtpe.py From fe411a80b1189c540bdd7a275d5afd39daa8161a Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Tue, 26 Aug 2025 18:23:07 -0400 Subject: [PATCH 24/26] fix the import in sca.py --- roman_imsim/sca.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roman_imsim/sca.py b/roman_imsim/sca.py index 5dea767b..9fbe5ef9 100644 --- a/roman_imsim/sca.py +++ b/roman_imsim/sca.py @@ -8,7 +8,7 @@ from galsim.image import Image -import roman_imsim.Effects as RomanEffects +import roman_imsim.effects as RomanEffects class RomanSCAImageBuilder(ScatteredImageBuilder): From 0346889132c0698472e749d7a39294c9bbae590c Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Wed, 27 Aug 2025 01:12:45 -0400 Subject: [PATCH 25/26] bug fix in read_noise.py: missed assignment for simple model --- roman_imsim/effects/read_noise.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roman_imsim/effects/read_noise.py b/roman_imsim/effects/read_noise.py index d001a5ee..baac8224 100644 --- a/roman_imsim/effects/read_noise.py +++ b/roman_imsim/effects/read_noise.py @@ -1,5 +1,7 @@ import os import fitsio as fio +import galsim +import galsim.roman as roman from . import RomanEffects from .utils import sca_number_to_file @@ -12,6 +14,7 @@ def __init__(self, params, base, logger, rng, rng_iter=None): def simple_model(self, image): self.logger.warning("Simple model will be applied for read noise.") + self.read_noise = galsim.GaussianNoise(self.rng, sigma=roman.read_noise) self.im_read = image.copy() image.addNoise(self.read_noise) self.im_read = image - self.im_read From 1ff7ee3d531c5cd6c07ee38a8163964cac1c5fc6 Mon Sep 17 00:00:00 2001 From: Yuedong Fang Date: Thu, 28 Aug 2025 16:59:29 -0400 Subject: [PATCH 26/26] * remove duplicates in detector_physics.py --- roman_imsim/detector_physics.py | 51 --------------------------------- 1 file changed, 51 deletions(-) diff --git a/roman_imsim/detector_physics.py b/roman_imsim/detector_physics.py index 7e58f8d4..a2028ae8 100644 --- a/roman_imsim/detector_physics.py +++ b/roman_imsim/detector_physics.py @@ -83,15 +83,6 @@ def __init__(self, params, visit, SCA): )[self.sca] self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix / 2), int(roman.n_pix / 2))) - self.WCS = roman.getWCS( - world_pos=galsim.CelestialCoord(ra=obseq_data.ob["ra"], dec=obseq_data.ob["dec"]), - PA=obseq_data.ob["pa"], - date=self.date, - SCAs=self.sca, - PA_is_FPA=True, - )[self.sca] - self.radec = self.WCS.toWorld(galsim.PositionI(int(roman.n_pix / 2), int(roman.n_pix / 2))) - class modify_image(object): """ @@ -121,8 +112,6 @@ def __init__(self, params, visit, sca, dither_from_file, sca_filepath=None, use_ if sca_filepath is not None: self.df = fio.FITS(sca_filepath + "/" + sca_number_to_file[self.pointing.sca]) print("------- Using SCA files --------") - self.df = fio.FITS(sca_filepath + "/" + sca_number_to_file[self.pointing.sca]) - print("------- Using SCA files --------") else: self.df = None print("------- Using simple detector model --------") @@ -139,16 +128,6 @@ def __init__(self, params, visit, sca, dither_from_file, sca_filepath=None, use_ "truth", self.get_path_name(use_galsim=use_galsim) ) - b = galsim.BoundsI(xmin=1, ymin=1, xmax=roman.n_pix, ymax=roman.n_pix) - old_filename = os.path.join(self.params["output"]["dir"], imfilename) - if not os.path.exists( - self.params["output"]["dir"].replace("truth", self.get_path_name(use_galsim=use_galsim)) - ): - os.mkdir(self.params["output"]["dir"].replace("truth", self.get_path_name(use_galsim=use_galsim))) - new_filename = os.path.join(self.params["output"]["dir"], imfilename).replace( - "truth", self.get_path_name(use_galsim=use_galsim) - ) - b = galsim.BoundsI(xmin=1, ymin=1, xmax=roman.n_pix, ymax=roman.n_pix) im = fio.FITS(old_filename)[-1].read() im = galsim.Image(im, bounds=b, wcs=self.pointing.WCS) @@ -160,7 +139,6 @@ def __init__(self, params, visit, sca, dither_from_file, sca_filepath=None, use_ force_cvz = True self.setup_sky(im, self.pointing, rng, visit * sca, force_cvz=force_cvz) - img, err, dq, sky_mean, sky_noise = self.add_effects(im, None, self.pointing, use_galsim=use_galsim) img, err, dq, sky_mean, sky_noise = self.add_effects(im, None, self.pointing, use_galsim=use_galsim) write_fits( @@ -172,7 +150,6 @@ def __init__(self, params, visit, sca, dither_from_file, sca_filepath=None, use_ self.pointing.sca, sky_mean=sky_mean, ) - write_fits(old_filename, new_filename, img, sky_noise, dq, self.pointing.sca, sky_mean=sky_mean) def get_path_name(self, use_galsim=False): @@ -562,10 +539,6 @@ def bfe(self, im): # The img is clipped by the saturation level here to cap the brighter fatter effect # and avoid unphysical behavior - array_pad = self.saturate(im.copy()).array[4:-4, 4:-4] # img of interest 4088x4088 - array_pad = np.pad( - array_pad, [(4 + nbfe, 4 + nbfe), (4 + nbfe, 4 + nbfe)], mode="symmetric" - ) # 4100x4100 array array_pad = self.saturate(im.copy()).array[4:-4, 4:-4] # img of interest 4088x4088 array_pad = np.pad( array_pad, [(4 + nbfe, 4 + nbfe), (4 + nbfe, 4 + nbfe)], mode="symmetric" @@ -697,8 +670,6 @@ def get_eff_sky_bg(self, pointing, radec): radec : World coordinate position of image """ - sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) - sky_level *= (1.0 + roman.stray_light_fraction) * roman.pixel_scale**2 sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) sky_level *= (1.0 + roman.stray_light_fraction) * roman.pixel_scale**2 @@ -739,9 +710,6 @@ def setup_sky(self, im, pointing, rng, rng_iter, force_cvz=False): self.dark_current_ = ( roman.dark_current * roman.exptime + self.df["DARK"][:, :].flatten() * roman.exptime ) - self.dark_current_ = ( - roman.dark_current * roman.exptime + self.df["DARK"][:, :].flatten() * roman.exptime - ) if self.df is None: self.gain = roman.gain else: @@ -755,8 +723,6 @@ def setup_sky(self, im, pointing, rng, rng_iter, force_cvz=False): radec = pointing.radec sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) sky_level *= 1.0 + roman.stray_light_fraction - sky_level = roman.getSkyLevel(pointing.bpass, world_pos=radec, date=pointing.date) - sky_level *= 1.0 + roman.stray_light_fraction # Make a image of the sky that takes into account the spatially variable pixel scale. Note # that makeSkyImage() takes a bit of time. If you do not care about the variable pixel # scale, you could simply compute an approximate sky level in e-/pix by multiplying @@ -864,8 +830,6 @@ def dark_current(self, im): # opt for numpy random geneator instead for speed self.im_dark = self.rng_np.poisson(dark_current_).reshape(im.array.shape).astype(im.dtype) im.array[:, :] += self.im_dark - self.im_dark = self.rng_np.poisson(dark_current_).reshape(im.array.shape).astype(im.dtype) - im.array[:, :] += self.im_dark # NOTE: Sky level and dark current might appear like a constant background that can be # simply subtracted. However, these contribute to the shot noise and matter for the @@ -960,11 +924,6 @@ def add_persistence(self, im, pointing): p_list = np.array([get_pointing(self.params, i, pointing.sca) for i in dither_list_selected]) dt_list = np.array([(pointing.date - p.date).total_seconds() for p in p_list]) p_pers = p_list[np.where((dt_list > 0) & (dt_list < roman.exptime * 10))] - dither_list_selected = dither_sca_array[dither_sca_array[:, 1] == pointing.sca, 0] - dither_list_selected = dither_list_selected[np.abs(dither_list_selected - pointing.visit) < 10] - p_list = np.array([get_pointing(self.params, i, pointing.sca) for i in dither_list_selected]) - dt_list = np.array([(pointing.date - p.date).total_seconds() for p in p_list]) - p_pers = p_list[np.where((dt_list > 0) & (dt_list < roman.exptime * 10))] if self.df is None: # iterate over previous exposures @@ -986,7 +945,6 @@ def add_persistence(self, im, pointing): x = x.clip(0) # remove negative stimulus im.array[:, :] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * roman.exptime - im.array[:, :] += galsim.roman.roman_detectors.fermi_linear(x.array, dt) * roman.exptime else: @@ -1106,8 +1064,6 @@ def interpix_cap(self, im, kernel=roman.ipc_kernel): num_grids = 4 # num_grids <= 8 grid_size = 4096 // num_grids - array_pad = im.array[4:-4, 4:-4] # it's an array instead of img - array_pad = np.pad(array_pad, [(5, 5), (5, 5)], mode="symmetric") # 4098x4098 array array_pad = im.array[4:-4, 4:-4] # it's an array instead of img array_pad = np.pad(array_pad, [(5, 5), (5, 5)], mode="symmetric") # 4098x4098 array @@ -1127,8 +1083,6 @@ def interpix_cap(self, im, kernel=roman.ipc_kernel): for i in range(3): tmp = (t.dot(K[j, i, :, :])).dot(t.T) # grid_sizexgrid_size K_pad[j, i, :, :] = np.pad(tmp, [(1, 1), (1, 1)], mode="symmetric") - tmp = (t.dot(K[j, i, :, :])).dot(t.T) # grid_sizexgrid_size - K_pad[j, i, :, :] = np.pad(tmp, [(1, 1), (1, 1)], mode="symmetric") for dy in range(-1, 2): for dx in range(-1, 2): @@ -1177,11 +1131,6 @@ def add_read_noise(self, im): self.rng_np.normal(loc=0.0, scale=read_noise).reshape(im.array.shape).astype(im.dtype) ) im.array[:, :] += self.im_read - read_noise = self.df["READ"][2, :, :].flatten() # flattened 4096x4096 array - self.im_read = ( - self.rng_np.normal(loc=0.0, scale=read_noise).reshape(im.array.shape).astype(im.dtype) - ) - im.array[:, :] += self.im_read # noise_array = self.rng_np.normal(loc=0., scale=read_noise) # 4088x4088 img