Post Views: 973

As project lead programmer, you need to re-run all programs in sequence to make sure that any updates to raw data or any revisions to programs can be reflected in TFL outputs. You can re-run all ADaM programs at dev side and then re-run all ADaM programs at qc side. Then re-run all TFL programs at dev side and later re-run all TFL programs at qc side. Easy right? However, there are several points that require your attention. Otherwise, errors may occur. This post will introduce two macros which can help you easily batch run your programs and prevent you from making certain mistakes.

Delete old outputs like ADaM datasets, TFL outputs, TFL datasets

Some tables may be removed from or added into Mockup file and thus table number will be changed. In this case, old TFL outputs or datasets should be deleted from our output folder. Or code responsible for producing TFL outputs have been commented and outputs will not be refreshed even we re-run our programs. All these wrong TFL outputs may be bundled together if they are not removed. To prevent this from happening, it is highly recommended for you to remove all ADaM datasets, TFL outputs, TFL datasets from both DEV side and QC side before you start re-run all programs. Commonly, all these outputs are placed in different folders. It will be annoying if you delete those files manually. Luckily, functions such as DOPEN, FILENAME, FDELETE, DREAD, MOPEN can enable us to manipulate files within a folder within DATA step.
Following function can retrieve filename of all files within a folder and then delete all files with extension such as .rtf, .log, .lst, .sas7bdat.

Click here to hide/show code


%macro m_fdel(pgm_path=);
data _null_;
length dref $8 fnm $80;

rc = filename(dref, “&pgm_path”);
filename dirref “&pgm_path”;

if rc = 0 then did = dopen(dref);

dnum = dnum(did);

do i = 1 to dnum;
fnm = dread(did, i);
fid = mopen(did, fnm);

if fid > 0 and (index(lowcase(fnm),’.log’)>0 or index(lowcase(fnm),’.lst’)>0
or index(lowcase(fnm),’.rtf’)>0 or index(lowcase(fnm),’.sas7bdat’)>0)
then do;
rc = filename(‘fref’,fnm, , ,’dirref’);
rc = fdelete(‘fref’);
end;
end;

rc = dclose(did);
run;
%mend m_fdel;

*Examples demonstrating how to delete files ;
%m_fdel(pgm_path= &study_path/dev/adam/log);
%m_fdel(pgm_path= &study_path/dev/adam/output);
%m_fdel(pgm_path= &study_path/qc/adam/log);
%m_fdel(pgm_path= &study_path/qc/adam/output);
%m_fdel(pgm_path= &study_path/dev/tfls/log);
%m_fdel(pgm_path= &study_path/dev/tfls/output);
%m_fdel(pgm_path= &study_path/qc/tfls/log);
%m_fdel(pgm_path= &study_path/qc/tfls/output);

Check if log files are clean

To make sure that code developed by individual programmers are robust, you may request that all log files should be clean and warning/error message is not allowed. Even some note message is forbidden. What will you do to check if all log files are clean? Following macro read log file from a folder into sas dataset and pick out warning message and then output the warning message into a lst file. It will check log files one by one and output all warning message as well as fullname of log files into lst file. In this way, you can easily find which programs require update from looking at the lst file.

Click here to hide/show code


%macro m_logchk(pgm_path =, out_path = , out_file = );
*Get name of all sas files;
data files(keep=fnm);
length dref $8 fnm $80;

rc = filename(dref, “&pgm_path”);
if rc = 0 then did = dopen(dref);

dnum = dnum(did);
do i = 1 to dnum;
fnm = dread(did, i);
fid = mopen(did, fnm);
if fid > 0 and index(lowcase(fnm),’.log’)>0 then output;
end;
rc = dclose(did);
run;

data _null_;
dsid = open(“files”);
nobs = attrn(dsid,’nobs’);
call symputx(“nobs”,nobs);
run;

%if %eval(&nobs) > 0 %then %do;

data _null_;
set files end = eof;
n = _n_;
if mfl ne 1 then mfl = 0;
call symputx(“pgm_name”||strip(put(n,best.)),strip(fnm));
if eof then call symputx(“no_prg”,strip(put(n,best.)));
run;

ods _all_ close;
ods listing file = “&out_path/&out_file..lst”;

%do i=1 %to &no_prg;

%let logcheck=%upcase(&pgm_path &&pgm_name&i);

%let existny = %Sysfunc(fileexist(“&pgm_path/&&pgm_name&i”));

%if &existny = 1 %then %do;
data _logchk;
length logcheck $2000;
infile “&pgm_path/&&pgm_name&i”;
input;

linenum=_n_;
logcheck = strip(_infile_);

if find(_infile_,’ERROR’)>0 or find(_infile_,’USER WARNING’)>0 then output;
else if find(_infile_,’WARNING:’)>0 and
not (index(upcase(_infile_),’WILL BE EXPIRING SOON’)>0 or
index(upcase(_infile_),’THE BASE PRODUCT PRODUCT WITH WHICH’)> 0 or
index(upcase(_infile_),’COMPRESSION WAS DISABLED FOR DATA SET’)> 0 or
index(upcase(_infile_),’SCHEDULED TO EXPIRE’)>0) then output;
else if find(_infile_,’NOTE:’)>0 or find(_infile_,’INFO:’)>0 then do;
if indexw(upcase(_infile_),’WARNING:’)>0 or
index(upcase(_infile_), ‘ERROR’)>0 or
indexw(upcase(_infile_),’MERGE’)>0 or
index(upcase(_infile_), ‘UNINITIALIZED’)>0 or
index(upcase(_infile_), ‘CATALOG IS OPENED FOR READ ONLY’)>0 or
index(upcase(_infile_), ‘FORMAT WAS TOO SMALL’)>0 or
index(upcase(_infile_), ‘VALUES HAVE BEEN CONVERTED TO’)>0 or
index(upcase(_infile_), ‘MISSING VALUES WERE’)>0 or
index(upcase(_infile_), ‘IS ALREADY USED OR NOT A VALID SAS NAME’)>0 or
index(upcase(_infile_), ‘IS ALREADY ON THE LIBRARY’)>0 or
index(upcase(_infile_), ‘DIVISION BY ZERO’)>0 or
index(upcase(_infile_), ‘MATHEMATICAL OPERATIONS COULD NOT’)>0 or
indexw(upcase(_infile_),’INVALID’)>0 or
indexw(upcase(_infile_),’OVERWRITTEN’)>0 or
index(upcase(_infile_), ‘NO OBSERVATIONS’)>0 or
index(upcase(_infile_), ‘THERE WERE 0 OBSERVATIONS’)>0 or
index(upcase(_infile_), ‘AT LEAST ONE W.D FORMAT’)>0 or
indexw(upcase(_infile_),’TRUNCATED’)>0 or
index(upcase(_infile_), ‘OUTSIDE THE AXIS’)>0 or
index(upcase(_infile_), ‘OBSERVATIONS WERE DELETED’)>0 OR
index(upcase(_infile_), ‘REPEATS OF BY VALUES’)>0 then output;
end;
run;
%end;
%else %if &existny = 0 %then %do;
data _logchk;
length logcheck $2000;
linenum = 1;
logcheck = “Log is missing”;
run;
%end;

proc report data=_logchk missing nowd headskip split=’|’;
columns (linenum logcheck);
define linenum / order ” width=6 right;
define logcheck / display “| |&logcheck” width=100 left flow;
run;
%end;
ods listing close;
ods listing;
%end;
%mend m_logchk;

Examples to demonstrate how to check log files;
%let path = &study_path;