= %sysfunc(dequote(&ds_in)); = %sysfunc(dequote(&var));

Transcription

1 / SUGI PAPER Adding Statistical Functionality to the DATA Step with FCMP BY: Stacey Christian and Jacques Rioux EXAMPLE 1: Recursive Algorithm for Segmenting Time Series Data / / Macro to get number of observations in a dataset Parameters: ds - name of dataset get nobs for nobs - output / %macro get_nobs_macro; %let ds = %sysfunc(dequote(&ds)); %let nobs=0; %let dsid=%sysfunc(open(&ds,i)); %if &dsid ^= 0 %then %do; %let nobs=%sysfunc(attrn(&dsid,nlobs)); %let rc=%sysfunc(close(&dsid)); % %mend get_nobs_macro; / Macro to append segment information to dataset Parameters: seg_ds - name of dataset append to first_obs - first row in segment last_obs - last row in segment error - approximation error for segment / %macro append_segment_macro; %let seg_ds = %sysfunc(dequote(&seg_ds)); data _temp_; start = &first_obs; end = &last_obs; error = &error; proc append base=&seg_ds data=_temp_ force; %mend append_segment_macro;

2 / Macro to perform a linear approximation of a segment of time series data using PROC REG. Parameters: ds_in - name of segment dataset first_obs - first row in segment last_obs - last row in segment var - variable to model interval - time interval Output Parameters: error - approximation error for segment / %macro linear_approximation_macro; %let ds_in %let var = %sysfunc(dequote(&ds_in)); = %sysfunc(dequote(&var)); data _temp1_; set &ds_in(firstobs=&first_obs obs=&last_obs); retain _TREND_ 0; _TREND_ = _TREND_ + 1; proc reg data=_temp1_ outest=_est_ noprint; model &var = _TREND_ / sse; output out=_reg_&first_obs._&last_obs. p=predict; quit; proc sql noprint; select _SSE_ into :ERROR from _est_; quit; %mend linear_approximation_macro; / FCMP Function wrappers for above macros / proc fcmp outlib=sasuser.examples.topdown; function get_nobs(ds $); nobs = 0; rc = run_macro( 'get_nobs_macro', ds, nobs ); return(nobs); subroutine append_segment(seg_ds $, start, end, error); first_obs = start; last_obs = rc = run_macro( 'append_segment_macro', seg_ds, first_obs, last_obs, error ); function linear_approximation(ds_in $, var $, interval $, first_obs, last_obs); error =.; rc = run_macro('linear_approximation_macro', ds_in, first_obs, last_obs, var, interval, error); return(error);

3 / set cmplib option so our functions can be found / options cmplib= sasuser.examples; / Top-Down Segmentation Function. Resursively splits data in half until segments meet threshold criteria. Parameters: data : time-series data to segment var : variable to model interval : interval of data start : first row (MISSING first time) end : last row (MISSING first time) error : error for current segment (MISSING first time) threshold : threshold for error improvement (percentage) min_seg : minimum observations for segment Output Parameters: seg_data - dataset to write segment info / proc fcmp outlib=sasuser.examples.topdown; subroutine segment_topdown(data $, seg_data $, var $, interval $, start_in, end_in, error_in, threshold, min_seg); /- initial setup -/ if (start_in eq.) then start = 1; else start=start_in; if (end_in eq.) then end = get_nobs(data); else end=end_in; /- calculate error on current segment -/ if (error_in eq.) then do; error = linear_approximation(data,var,interval,start,end); else do; error = error_in; put "segment_topdown called with" start= end= error= ; /- calculate mid point -/ mid = start + floor((end-start)/2); /- calculate error on LEFT SEGMENT -/ left_error = linear_approximation(data,var,interval,start,mid); /- calculate error on RIGHT SEGMENT -/ right_error = linear_approximation(data,var,interval,mid+1,end); /- calculate combined error improvement -/ combined_error = left_error + right_error; improvement = (error - combined_error) / error; splitlength = mid-start; put combined_error= improvement=; / recursively split segment / if (improvement > threshold AND splitlength/2 >= min_seg) then do;

4 / left segment / call segment_topdown(data,seg_data,var,interval, start,mid,left_error,threshold,min_seg); / right segment / call segment_topdown(data,seg_data,var,interval, mid+1,end,right_error,threshold,min_seg); / else store current segment / else do; call append_segment(seg_data,start,end,error); / make sure there are no old datasets hanging around / proc datasets library=work memtype=data nolist; delete _reg_: segds_: ; quit; libname sugi "\\d71235\public\sugi"; data _NULL_; call segment_topdown("sugi.snp", "work.segds_reg_15", "close", "daily",.,.,., 0.15, 10); call segment_topdown("sugi.snp", "work.segds_reg_20", "close", "daily",.,.,., 0.20, 10);

5 / SUGI PAPER Adding Statistical Functionality to the DATA Step with FCMP BY: Stacey Christian and Jacques Rioux EXAMPLE 1: Recursive algorithm with PROC REG to segment time series data Code for plotting piecewise linear approximations versus original data / / Macro to plot the piecewise linear approximations versus original data. Parameters: seg_ds - name of segment dataset var - variable that was modeled interval - interval of data prefix - ds name prefix for regression data / %macro plot_segments_macro; %let seg_ds = %sysfunc(dequote(&seg_ds)); %let prefix = %sysfunc(dequote(&prefix)); %let var = %sysfunc(dequote(&var)); %let interval = %sysfunc(dequote(&interval)); / combined segments in one dataset / %let nobs=0; %let ds=&seg_ds; %get_nobs_macro; %do i=1 %to &nobs; data _NULL_; set &seg_ds(firstobs=&i obs=&i); length dsname $ 256; dsname = trim(left("&prefix")) trim(left(start)) "_" trim(left(end)); call symputx("ds&i", trim(dsname)); % data _combined_; set &ds1; %do i=2 %to &nobs; proc append base= _combined_ data = &&ds&i; % data _combined_; set _combined_; label PREDICT="Linear Approximation of &var"; ods graphics on; ods html; proc sgplot data=_combined_; series x=date y=&var / lineattrs=(pattern=solid thickness=0.1); series x=date y=predict / lineattrs=(pattern=solid thickness=1); ods graphics off; ods html close; %mend plot_segments_macro;

6 / FCMP Function wrapper for plotting macro / proc fcmp outlib=sasuser.examples.topdown; subroutine plot_segments(seg_ds $, var $, interval $, prefix $); rc = run_macro('plot_segments_macro', seg_ds, var, interval, prefix); / Run plotting code from DATA step / options cmplib=(sasuser.examples) nomprint; data _NULL_; call plot_segments("segds_reg_15","close","daily","_reg_"); call plot_segments("segds_reg_20","close","daily","_reg_");

7 / SUGI PAPER Adding Statistical Functionality to the DATA Step with FCMP BY: Stacey Christian and Jacques Rioux EXAMPLE 2: Iteratively Reweighted Least Squares Algorithm Data creation for example / // / Macro to create a sample simulated data set: / / The base survival model is Exponential with mean=10 / / There are 3 categorical covariates: X1, X2 and X3 / / X1 = 0 or 1 / / X2 = 0 or 0.4 or 0.9 / / X3 = -1 or 1 / / The model simulated corresponds to 0.1X X X3 / / The variables in the dataset are X1, X2, X3,Quarter, and Count / // %macro generate_survival_data_macro; %let dsname = %sysfunc(dequote(&dsname)); DATA &dsname; array AX1[2] _temporary_ (0 1); array AX2[3] _temporary_ ( ); array AX3[2] _temporary_ (-1 1); array ATime_[13] Time_0-Time_12; do i = 1 to dim(ax1); do j = 1 to dim(ax2); do k = 1 to dim(ax3); X1 = AX1[i]; X2 = AX2[j]; X3 = AX3[k]; Effect = exp(0.1 X X X3); do l = 1 to 13; ATime_[l] = 0; do l = 1 to 1000; Time = 10/Effect rand("exponential"); Quarters = ceil(4time); do m = 1 to 13; if Quarters > m-1 then do; ATime_[m] = ATime_[m] + 1; lastsurv = 1; do l = 1 to 13; Quarter = l - 1; Count = ATime_[l]; Surv = 1 - cdf("exponential",(l-1)/40); Prob = Surv/lastSurv; lastsurv = Surv; output;

8 drop Effect Surv Prob; drop i j k l m quarters lastsurv time time_:; RUN; %mend generate_survival_data_macro; // / run macro and create data for Example 2 / // %let dsname= mdphdata; %generate_survival_data_macro;

9 / SUGI PAPER Adding Statistical Functionality to the DATA Step with FCMP BY: Stacey Christian and Jacques Rioux EXAMPLE 2: Iteratively Reweighted Least Squares Algorithm / / Macro to prepare data for the Iteratively Weighted Least Squares Algorithm. Creates indicator variables for each time interval and calculates log_log_pij = log(-log(prob of survival)) Parameters: data : the dataset on which to run the PH model count : name of the count variable time : name of the time variable regressors : list of regression variables outdata : output dataset to be created intervalcount : number of time intervals (OUT) intervalvars : Interval_1-Interval_N (OUT) / %macro prepare_phdata_macro; %let data = %sysfunc( dequote(&data )); %let count = %sysfunc( dequote(&count )); %let time = %sysfunc( dequote(&time )); %let regressors = %sysfunc( dequote(&regressors )); %let outdata = %sysfunc( dequote(&outdata )); %let intervalvars = %sysfunc( dequote(&intervalvars )); proc sort data=&data out =&outdata; by &regressors &time; proc sql noprint; select distinct &time into :times separated by " " from &outdata order by &time ; select count(distinct &time) into :timecount from &outdata order by &time ; select max(&time) into :MaxTime from &outdata ; quit; %let IntervalCount = %eval(&timecount-1); data &outdata; set &outdata; retain LastCount 0 LastTime &MaxTime; array TimeIndicator[&IntervalCount] Interval_1-Interval_%trim(&IntervalCount); array Times[&TimeCount] _TEMPORARY_ (&Times); if &Time > LastTime then do;

10 do i = 1 to &IntervalCount; if &Time = Times[i+1] then do; TimeIndicator[i] = 1; else do; TimeIndicator[i] = 0; Nij = LastCount; Pij = &Count/LastCount; Log_Log_Pij = log(-log(&count/lastcount)); Weight = 1.0; output; LastCount = &Count; LastTime = &Time; drop &Count i LastTime LastCount; %let intervalvars = Interval_1-Interval_%trim(&IntervalCount); %mend prepare_phdata_macro; / Macro that uses proc REG to run regression Parameters: data : the dataset on which to run the regression params : output dataset for the fitted parameters dependent : name of the dependent variable independent : space delimited list of independent variables weight : name of the weighting variable / %macro run_regression_macro; %let data = %sysfunc( dequote(&data) ); %let params = %sysfunc( dequote(&params) ); %let dependent = %sysfunc( dequote(&dependent) ); %let independent = %sysfunc( dequote(&independent) ); %let weight = %sysfunc( dequote(&weight) ); proc reg data=&data outest=&params NOPRINT; model &dependent = &independent/noint; weight &weight; quit; data &params; set &params; keep &independent; %mend run_regression_macro;

11 / Macro to recompute the regression weight at every iteration in data set passed. Parameters: data : the dataset on which to reweight params : output parameter data set from proc reg regressors : name of the regression variables weight : name of the weighting variable / %macro update_weights_macro; %let data = %sysfunc( dequote(&data) ); %let params = %sysfunc( dequote(&params) ); %let regressors = %sysfunc( dequote(&regressors)); %let weight = %sysfunc( dequote(&weight)); data &params._params; set &params; / set Param_x1 = x1; Param_x2 = x2;... / %let nvar = 1; %let Var = %scan(&regressors, &nvar, ' '); %do %while ("&Var" ne ""); Param_&Var. = &Var; %let nvar = %eval(&nvar + 1); %let Var = %scan(&regressors, &nvar, ' '); % / set Param_Interval_1 = Interval_1;... / %do i = 1 %to &intervalcount; Param_Interval_&i = Interval_&i; % keep Param_:; / merge params with data / proc sql noprint; create table _temp_ as select from &data, &params._params; quit; / re-calculate weight / data &data; set _temp_; retain LastSj 1 lasttime 0; array Param_Interval[&intervalCount] Param_Interval_1 - Param_Interval_&intervalCount; array Interval[&intervalCount] Interval_1 - Interval_&intervalCount; Pj = 0; do i = 1 to &intervalcount.; Pj = Pj + exp(-exp(param_interval[i])) Interval[i]; if _N_ = 1 then do; Sj = Pj; else do; if &time < lasttime then do; Sj = Pj ; else do; Sj = LastSj Pj; / ExpXB = exp(param_x1 x1 + Param_x2 x Param_xn xn); / ExpXB = exp(0 %let nvar = 1;

12 ); %let Var = %scan(&regressors, &nvar, ' '); %do %while ("&Var" ne ""); + Param_&Var. &Var %let nvar = %eval(&nvar + 1); %let Var = %scan(&regressors, &nvar, ' '); % &Weight = ( 1 - PjExpXB ) / Nij / PjExpXB / ExpXB2 / log(pj)2; LastSj = Sj; LastTime = &time; drop LastTime LastSj i Param_:; %mend update_weights_macro;

13 / FCMP Functions Wrappers for macros above / proc fcmp outlib=sasuser.example2.mindist; subroutine prepare_phdata(data $, count $, time $, regressors $, outdata $, intervalcount, intervalvars $); outargs intervalcount, intervalvars; intervalvars = ""; intervalcount =.; rc = run_macro ( 'prepare_phdata_macro', data, count, time, regressors, outdata, intervalcount, intervalvars ); subroutine run_regression( data $, dependent $, independent $, weight $, params $, paramarray[]); outargs paramarray; array tmparray[1] _temporary_; call dynamic_array(tmparray,dim(paramarray)); rc = RUN_MACRO ('run_regression_macro', data, params, dependent, independent, weight) ; rc = read_array(params, tmparray); do i = 1 to dim(paramarray); paramarray[i] = tmparray[1,i]; subroutine update_weights( data $, params $, regressors $, weight $, intervalcount); rc = RUN_MACRO ('update_weights_macro', data, params, regressors, weight, intervalcount); function calc_max_relative_diff(params1[],params2[]); outargs params1; maxrelativedifference = 0; numparams = DIM(params1); do i = 1 to numparams; maxrelativedifference = max( maxrelativedifference, abs((params1[i]-params2[i])/params2[i])); params1[i] = params2[i]; return(maxrelativedifference);

14 // / / / The main algorithm to implement the Iteratively Reweighted / / Least Square Estimator. / / / / Parameters: / / data : the dataset on which to run the PH model / / outdata : output dataset for the fitted parameters / / count : name of the count variable / / time : name of the time variable / / regressors : name of the weighting variable / / paramds : fitted parameters dataset name / / / // options cmplib=sasuser.example2; %let numparams = 15; %let data = "work.mdphdata"; %let prepareddata = "work.outdata"; %let paramdata = "work.phresults"; %let regressors = "x1 x2 x3"; %let regcount = 3; %let depvar = "log_log_pij"; %let weightvar = "Weight"; data _NULL_; /- initial data preparation -/ length intervalvars $ 128; intervalcount = 0; intervalvars = ""; call prepare_phdata( &data, "Count", "Quarter", &regressors, &prepareddata, intervalcount, intervalvars ); independentvars = &regressors " " intervalvars; array params1[&numparams] _temporary_; array params2[&numparams] _temporary_; /- initial regression run with initial weights -/ call run_regression( &prepareddata, &depvar,independentvars, &weightvar, &paramdata, params1 ); /- iterative loop -/ maxrelativedifference = 1; do while( maxrelativedifference > ); call update_weights(&prepareddata, &paramdata, &regressors, &weightvar, intervalcount); call run_regression( &prepareddata, &depvar,independentvars, &weightvar, &paramdata, params2 ); maxrelativedifference = calc_max_relative_diff(params1,params2); put maxrelativedifference=; call update_weights(&prepareddata, &paramdata, &regressors, &weightvar, intervalcount);

15 / SUGI PAPER Adding Statistical Functionality to the DATA Step with FCMP BY: Stacey Christian and Jacques Rioux EXAMPLE 3: Proportional Hazard Model Simulation / %macro append_data_macro; %let data = %sysfunc( dequote(&data) ); %let base = %sysfunc( dequote(&base) ); proc append base = &base data = &data force; quit; %mend append_data_macro; proc fcmp outlib=sasuser.example2.sim ; subroutine simulate_phdata( dsname $); rc = run_macro('generate_survival_data_macro', dsname); subroutine append_data( base $, data $ ); rc = run_macro('append_data_macro', base, data ); subroutine fit_ph_model(data $, parmdataout $, countvar $, timevar $, depvar $, weightvar $, regressors $); /- initial data preparation -/ length intervalvars $ 128; intervalcount = 0; intervalvars = ""; call prepare_phdata( data, "Count", "Quarter", regressors, "_prepdat_", intervalcount, intervalvars ); independentvars = regressors " " intervalvars; /- create param arrays dynamically / numparams = intervalcount + count(regressors, " ")+1; array params1[1] _temporary_; array params2[1] _temporary_; call dynamic_array(params1, numparams ); call dynamic_array(params2, numparams ); /- initial regression run with initial weights -/ call run_regression( "_prepdat_", depvar, independentvars, weightvar, parmdataout, params1 ); /- iterative loop -/ maxrelativedifference = 1; do while( maxrelativedifference > ); call update_weights("_prepdat_", parmdataout, regressors, weightvar, intervalcount); call run_regression( "_prepdat_", depvar, independentvars, weightvar, parmdataout, params2 ); maxrelativedifference = calc_max_relative_diff(params1,params2); put maxrelativedifference=;

16 call update_weights("_prepdat_", parmdataout, regressors, weightvar, intervalcount); / Run the simulation / options cmplib=(sasuser.example2); %let NSIM=20; proc fcmp; do i=1 to &NSIM; put ""; put "Simulation" i; put ""; call simulate_phdata("work.simdata"); call fit_ph_model("work.simdata", "work.params", "Count", "Quarter", "log_log_pij", "Weight", "x1 x2 x3"); call append_data("work.simresults", "work.params"); proc summary data= work.simresults; var X1 X2 X3 Interval_1-Interval_12; output out = work.simsummary mean = meanx1 meanx2 meanx3 meaninterval_1-meaninterval_12 var = varx1 varx2 varx3 varinterval_1-varinterval_12 ;