Skip to content
Open

Eso #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
852a375
Felix patch to make pipeline work
teuben Jun 19, 2018
ea1804c
for getFigure/Thumbnail, return None if figure doesn't exist AND plot…
mpound Sep 18, 2018
2f4a969
changing over to use new noplot member variable
mpound Sep 19, 2018
e881a04
changing over to use new noplot member variable
mpound Sep 19, 2018
32476bb
changing over to use new noplot member variable
mpound Sep 19, 2018
38afae6
changing over to use new noplot member variable
mpound Sep 19, 2018
d0943b2
changing over to use new noplot member variable
mpound Sep 19, 2018
f7f125d
changing over to use new noplot member variable
mpound Sep 19, 2018
0490bae
changing over to use new noplot member variable
mpound Sep 19, 2018
8f3673e
changing over to use new noplot member variable
mpound Sep 19, 2018
a606436
changing over to use new noplot member variable
mpound Sep 19, 2018
64878e4
changing over to use new noplot member variable
mpound Sep 19, 2018
13a153d
a few more tasks using noplot
mpound Sep 19, 2018
83504cc
this task does not create plots but inserted the noplot into SummaryE…
mpound Sep 19, 2018
ce5b58e
changing over to use new noplot member variable, and remove unused va…
mpound Sep 19, 2018
28bfd8c
pvslice_at and pvcorr_at using noplot
mpound Sep 19, 2018
2210205
changing over to use new noplot member variable
mpound Sep 19, 2018
ea55d3f
changing over to use new noplot member variable
mpound Sep 19, 2018
7a79ad6
changing over to use new noplot member variable
mpound Sep 19, 2018
0a9b24f
pca and overlapintegral using noplot
mpound Sep 19, 2018
5150642
moment_at using noplot
mpound Sep 19, 2018
554f0fa
changing over to use new noplot member variable
mpound Sep 19, 2018
0802ab0
changing over to use new noplot member variable
mpound Sep 19, 2018
5035081
linesegment_at using noplot
mpound Sep 19, 2018
eba7ff6
lineid_at using noplot
mpound Sep 19, 2018
046e2d4
changing over to use new noplot member variable
mpound Sep 19, 2018
d45071e
small bug fixes after runa1 testing
mpound Sep 20, 2018
acf32ec
more bugfixes from runa1
mpound Sep 20, 2018
6cca839
more bugfixes from runa1
mpound Sep 21, 2018
adaabd3
changing over to use new noplot member variable
mpound Sep 21, 2018
5d15e88
_noplot forgotten for admit.xml
teuben Oct 2, 2018
ac5aa21
fix getimagefile() in case where lower level return value is None, i.…
mpound Oct 5, 2018
3e400a9
show() must also check for MultiImages, since an Image_BDP does not a…
mpound Oct 5, 2018
21da053
catch a keyerror
mpound Oct 5, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 141 additions & 61 deletions admit/Summary.py
100644 → 100755

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion admit/at/BDPIngest_AT.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import admit.util.utils as utils
from admit.util.Table import Table
from admit.Summary import SummaryEntry
#import admit.util.PlotControl as PlotControl


class BDPIngest_AT(AT):
Expand Down Expand Up @@ -110,4 +111,4 @@ def run(self):
table.addRow(["Associated File",f])
table.description = "Information about the ingested BDP"
taskargs = "file=%s" % self.getkey('file')
self._summary["bdpingest"] = SummaryEntry(table.serialize(),"BDPIngest_AT",self.id(True),taskargs)
self._summary["bdpingest"] = SummaryEntry(table.serialize(),"BDPIngest_AT",self.id(True),taskargs,noplot=True)
24 changes: 17 additions & 7 deletions admit/at/ContinuumSub_AT.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from admit.bdp.LineList_BDP import LineList_BDP
from admit.bdp.LineSegment_BDP import LineSegment_BDP
import admit.util.utils as utils
import admit.util.PlotControl as PlotControl
import admit.util.filter.Filter1D as Filter1D
from admit.util.AdmitLogging import AdmitLogging as logging
import numpy as np
Expand Down Expand Up @@ -207,19 +208,28 @@ def run(self):
rdata = casautil.getdata(self.dir(f3)).data
logging.regression("CSUB: %f %f" % (rdata.min(),rdata.max()))

# Create two output images for html and their thumbnails, too
implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
implot.plotter(rasterfile=f3,figname=f3,colorwedge=True)
figname = implot.getFigure(figno=implot.figno,relative=True)
thumbname = implot.getThumbnail(figno=implot.figno,relative=True)
b2.setkey("image", Image(images={bt.CASA:f2}))
b3.setkey("image", Image(images={bt.CASA:f3, bt.PNG : figname}))

# Create two output images for html and their thumbnails, too
if self._plot_mode == PlotControl.NOPLOT:
figname = "not created"
thumbname = "not created"
imcaption = "not created"
b3.setkey("image", Image(images={bt.CASA:f3}))
noplot = True
else:
implot = ImPlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
implot.plotter(rasterfile=f3,figname=f3,colorwedge=True)
figname = implot.getFigure(figno=implot.figno,relative=True)
thumbname = implot.getThumbnail(figno=implot.figno,relative=True)
b3.setkey("image", Image(images={bt.CASA:f3, bt.PNG : figname}))
noplot = False
dt.tag("implot")

if len(ch) > 0:
taskargs = "pad=%d fitorder=%d contsub=%s" % (pad,fitorder,str(contsub))
imcaption = "Continuum map"
self._summary["continuumsub"] = SummaryEntry([figname,thumbname,imcaption],"ContinuumSub_AT",self.id(True),taskargs)
self._summary["continuumsub"] = SummaryEntry([figname,thumbname,imcaption],"ContinuumSub_AT",self.id(True),taskargs,noplot=noplot)

dt.tag("done")
dt.end()
Expand Down
40 changes: 27 additions & 13 deletions admit/at/CubeSpectrum_AT.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import admit.util.Image as Image
from admit.util import APlot
import admit.util.utils as utils
import admit.util.PlotControl as PlotControl
from admit.util.AdmitLogging import AdmitLogging as logging

from copy import deepcopy
Expand Down Expand Up @@ -267,9 +268,10 @@ def run(self):
planes = range(npos) # labels for the tables (placeholder)
images = {} # png's accumulated

noplot = True
for i in range(npos): # loop over pos, they can have mixed types now
sd = []
caption = "Spectrum"
imcaption = "Spectrum"
xpos = pos[i][0]
ypos = pos[i][1]
if type(xpos) != type(ypos):
Expand All @@ -282,7 +284,7 @@ def run(self):
cbox = '(%d,%d,%d,%d)' % (xpos,ypos,xpos,ypos)
# use extend here, not append, we want individual values in a list
sd.extend([xpos,ypos,cbox])
caption = "Average Spectrum at %s" % cbox
imcaption = "Average Spectrum at %s" % cbox
if False:
# this will fail on 3D cubes (see CAS-7648)
imval[i] = casa.imval(self.dir(fin),box=box)
Expand All @@ -291,12 +293,12 @@ def run(self):
# another approach is the ia.getprofile(), see CubeStats, this will
# also integrate over regions, imval will not (!!!)
region = 'centerbox[[%dpix,%dpix],[1pix,1pix]]' % (xpos,ypos)
caption = "Average Spectrum at %s" % region
imcaption = "Average Spectrum at %s" % region
imval[i] = casa.imval(self.dir(fin),region=region)
elif type(xpos)==str:
# this is tricky, to stay under 1 pixel , or you get a 2x2 back.
region = 'centerbox[[%s,%s],[1pix,1pix]]' % (xpos,ypos)
caption = "Average Spectrum at %s" % region
imcaption = "Average Spectrum at %s" % region
sd.extend([xpos,ypos,region])
imval[i] = casa.imval(self.dir(fin),region=region)
else:
Expand Down Expand Up @@ -354,14 +356,22 @@ def run(self):
else:
title = '%s %d @ %s,%s' % (bdp_name,i,xpos,ypos) # or use box, once we allow non-points

myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir())
ylab = 'Flux (%s)' % unit
p1 = "%s_%d" % (bdp_name,i)
myplot.plotter(x,y,title,p1,xlab=xlab,ylab=ylab,thumbnail=True)
# Why not use p1 as the key?
ii = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True)
thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True)
sd.extend([ii, thumbname, caption, fin])
if self._plot_mode == PlotControl.NOPLOT:
figname = "not created"
thumbname = "not created"
imcaption = "not created"
noplot = True
else:
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode, abspath=self.dir())
ylab = 'Flux (%s)' % unit
p1 = "%s_%d" % (bdp_name,i)
myplot.plotter(x,y,title,p1,xlab=xlab,ylab=ylab,thumbnail=True)
# Why not use p1 as the key?
figname = images["pos%d" % i] = myplot.getFigure(figno=myplot.figno,relative=True)
thumbname = myplot.getThumbnail(figno=myplot.figno,relative=True)
noplot = False

sd.extend([figname, thumbname, imcaption, fin])
self.spec_description.append(sd)

logging.regression("CSP: %s" % str(smax))
Expand All @@ -385,7 +395,11 @@ def run(self):
# SummaryEntry([[data for spec1]], "CubeSpectrum_AT",taskid)
# For multiple spectra this is
# SummaryEntry([[data for spec1],[data for spec2],...], "CubeSpectrum_AT",taskid)
self._summary["spectra"] = SummaryEntry(self.spec_description,"CubeSpectrum_AT",self.id(True))

# @todo if range(npos) is [] don't create a summary entry
# so that check against None in Summary.py does the right thing,
# although len(npos) == 0 is trapped earlier so perhaps not necessary
self._summary["spectra"] = SummaryEntry(self.spec_description, "CubeSpectrum_AT", self.id(True), noplot=noplot)
taskargs = "pos="+str(pos)
taskargs += '&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; &nbsp;&nbsp;&nbsp; <span style="background-color:white">&nbsp;' + fin.split('/')[0] + '&nbsp;</span>'
for v in self._summary:
Expand Down
122 changes: 84 additions & 38 deletions admit/at/CubeStats_AT.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from admit.util.segmentfinder import ADMITSegmentFinder
from admit.Summary import SummaryEntry
import admit.util.casautil as casautil
import admit.util.PlotControl as PlotControl
from admit.util.AdmitLogging import AdmitLogging as logging

from copy import deepcopy
Expand Down Expand Up @@ -400,24 +401,34 @@ def run(self):
y4 = np.zeros(len(minval))
y5 = y1-y4
y = [y1,y2,y3,y4]
title = 'CubeStats: ' + bdp_name+'_0'
xlab = 'Channel'
ylab = 'log(Peak,Noise,Peak/Noise)'
labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)']
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]]
myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)

image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0")
b2.addimage(image0,"im0")
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
imfile = "not created"
thumbfile = "not created"
else:
noplot = False
title = 'CubeStats: ' + bdp_name+'_0'
xlab = 'Channel'
ylab = 'log(Peak,Noise,Peak/Noise)'
labels = ['log(peak)','log(rms noise)','log(peak/noise)','log(|minval|)']
myplot = APlot(ptype=self._plot_type,pmode=self._plot_mode,abspath=self.dir())
segp = [[chans[0],chans[nchan-1],math.log10(sigma0),math.log10(sigma0)]]
myplot.plotter(chans,y,title,bdp_name+"_0",xlab=xlab,ylab=ylab,segments=segp,labels=labels,thumbnail=True)
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)

image0 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_0")
b2.addimage(image0,"im0")

if use_ppp:
# new trial for Lee
title = 'PeakSum: (numsigma=%.1f)' % (numsigma)
ylab = 'Jy*N_ppb'
myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
# new trial for Lee
title = 'PeakSum: (numsigma=%.1f)' % (numsigma)
ylab = 'Jy*N_ppb'
myplot.plotter(chans,[peaksum],title,bdp_name+"_00",xlab=xlab,ylab=ylab,thumbnail=False)

if True:
# hack ascii table
Expand All @@ -441,24 +452,29 @@ def run(self):
caption += " green: noise per channel,"
caption += " blue: peak value per channel,"
caption += " red: peak/noise per channel)."
self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True))
self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["spectra"] = SummaryEntry([0, 0, str(specbox), 'Channel', imfile, thumbfile , caption, fin], "CubeStats_AT", self.id(True),noplot=noplot)

self._summary["chanrms"] = SummaryEntry([float(sigma0), fin], "CubeStats_AT", self.id(True),noplot=noplot)

# @todo Will imstat["max"][0] always be equal to s['datamax']? If not, why not?
if 'datamax' in s:
self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["dynrange"] = SummaryEntry([float(s['datamax']/sigma0), fin], "CubeStats_AT", self.id(True),noplot=noplot)
else:
self._summary["dynrange"] = SummaryEntry([float(imstat0["max"][0]/sigma0), fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True))
self._summary["datamean"] = SummaryEntry([imstat0["mean"][0], fin], "CubeStats_AT", self.id(True),noplot=noplot)

title = bdp_name + "_1"
xlab = 'log(Peak,Noise,P/N)'
myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
title = bdp_name + "_1"
xlab = 'log(Peak,Noise,P/N)'
myplot.histogram([y1,y2,y3],title,bdp_name+"_1",xlab=xlab,thumbnail=True)

imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1")
b2.addimage(image1,"im1")
imfile = myplot.getFigure(figno=myplot.figno,relative=True)
thumbfile = myplot.getThumbnail(figno=myplot.figno,relative=True)
image1 = Image(images={bt.PNG:imfile},thumbnail=thumbfile,thumbnailtype=bt.PNG,description="CubeStats_1")
b2.addimage(image1,"im1")

# note that the 'y2' can have been clipped, which can throw off stats.robust()
# @todo should set a mask for those.
Expand All @@ -470,7 +486,11 @@ def run(self):
y2_mean = ry2.mean()
y2_std = ry2.std()
if n>9: logging.debug("NORMALTEST2: %s" % str(scipy.stats.normaltest(ry2)))
myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
myplot.hisplot(y2,title,bdp_name+"_2",xlab=xlab,gauss=[y2_mean,y2_std],thumbnail=True)

title = bdp_name + "_3"
xlab = 'log(diff[Noise])'
Expand All @@ -481,7 +501,11 @@ def run(self):
dy2_mean = rdy2.mean()
dy2_std = rdy2.std()
if n>9: logging.debug("NORMALTEST3: %s" % str(scipy.stats.normaltest(rdy2)))
myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
myplot.hisplot(dy2,title,bdp_name+"_3",xlab=xlab,gauss=[dy2_mean,dy2_std],thumbnail=True)


title = bdp_name + "_4"
Expand All @@ -491,7 +515,11 @@ def run(self):
y3_mean = ry3.mean()
y3_std = ry3.std()
if n>9: logging.debug("NORMALTEST4: %s" % str(scipy.stats.normaltest(ry3)))
myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
myplot.hisplot(y3,title,bdp_name+"_4",xlab=xlab,gauss=[y3_mean,y3_std],thumbnail=True)

title = bdp_name + "_5"
xlab = 'log(diff[Signal/Noise)])'
Expand All @@ -501,7 +529,11 @@ def run(self):
dy3_mean = rdy3.mean()
dy3_std = rdy3.std()
if n>9: logging.debug("NORMALTEST5: %s" % str(scipy.stats.normaltest(rdy3)))
myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
myplot.hisplot(dy3,title,bdp_name+"_5",xlab=xlab,gauss=[dy3_mean,dy3_std],thumbnail=True)


title = bdp_name + "_6"
Expand All @@ -511,7 +543,11 @@ def run(self):
y5_mean = ry5.mean()
y5_std = ry5.std()
if n>9: logging.debug("NORMALTEST6: %s" % str(scipy.stats.normaltest(ry5)))
myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
else:
noplot = False
myplot.hisplot(y5,title,bdp_name+"_6",xlab=xlab,gauss=[y5_mean,y5_std],thumbnail=True)

logging.debug("LogPeak: m,s= %f %f min/max %f %f" % (y1.mean(),y1.std(),y1.min(),y1.max()))
logging.debug("LogNoise: m,s= %f %f %f %f min/max %f %f" % (y2.mean(),y2.std(),y2_mean,y2_std,y2.min(),y2.max()))
Expand All @@ -534,12 +570,18 @@ def run(self):
s = np.pi * ( smax * (z0**gamma) )**2
cmds = ["grid", "axis equal"]
title = "Peak Points per channel"
pppimage = bdp_name + '_ppp'
myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True)
pppimage = myplot.getFigure(figno=myplot.figno,relative=True)
pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True)
caption = "Peak point plot: Locations of per-channel peaks in the image cube " + fin
self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True))
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
pppimage = "not created"
pppthumbnail = "not created"
else:
noplot = False
pppimage = bdp_name + '_ppp'
myplot.scatter(xpos,ypos,title=title,figname=pppimage,size=s,color=chans,cmds=cmds,thumbnail=True)
pppimage = myplot.getFigure(figno=myplot.figno,relative=True)
pppthumbnail = myplot.getThumbnail(figno=myplot.figno,relative=True)
self._summary["peakpnt"] = SummaryEntry([pppimage, pppthumbnail, caption, fin], "CubeStats_AT", self.id(True),noplot=noplot)
dt.tag("plotting")

# making PeakStats plot
Expand All @@ -552,7 +594,11 @@ def run(self):
ylab = 'FWHM (channels)'
pppimage = bdp_name + '_peakstats'
cval = mval
myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False)
if self._plot_mode == PlotControl.NOPLOT:
noplot = True
pppimage = "not created"
else:
myplot.scatter(pval,wval,title=title,xlab=xlab,ylab=ylab,color=cval,figname=pppimage,thumbnail=False)
dt.tag("peakstats")


Expand Down
Loading