diff --git a/.coverage b/.coverage
new file mode 100644
index 0000000000000000000000000000000000000000..e61eed4b7c43a3f551e07789589a565c8682ff7d
Binary files /dev/null and b/.coverage differ
diff --git a/.coverage.DESKTOP-ATMEKSV.24064.XFYCqtPx b/.coverage.DESKTOP-ATMEKSV.24064.XFYCqtPx
new file mode 100644
index 0000000000000000000000000000000000000000..e1f7e6f75f5aa9a7a33884193b67b7cc22082200
Binary files /dev/null and b/.coverage.DESKTOP-ATMEKSV.24064.XFYCqtPx differ
diff --git a/Outputs_SeqPosteriorComparison/posterior/Z.npy b/Outputs_SeqPosteriorComparison/posterior/Z.npy
new file mode 100644
index 0000000000000000000000000000000000000000..8d89efa6714257ec2d867aa5eb95b7f23b915010
Binary files /dev/null and b/Outputs_SeqPosteriorComparison/posterior/Z.npy differ
diff --git a/docs/diagrams/.$Structure_BayesInf.drawio.bkp b/docs/diagrams/.$Structure_BayesInf.drawio.bkp
new file mode 100644
index 0000000000000000000000000000000000000000..cd3914505bc5b8bdc582e24e4edcce7eda04e86b
--- /dev/null
+++ b/docs/diagrams/.$Structure_BayesInf.drawio.bkp
@@ -0,0 +1,67 @@
+<mxfile host="Electron" modified="2024-04-10T08:40:20.425Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/22.1.11 Chrome/114.0.5735.289 Electron/25.9.8 Safari/537.36" etag="ww-3i4-3angrh_Lyrbhs" version="22.1.11" type="device">
+ <diagram name="Page-1" id="efOe0Jku58RX-i1bv-3b">
+ <mxGraphModel dx="1434" dy="956" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+ <root>
+ <mxCell id="0" />
+ <mxCell id="1" parent="0" />
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-1" value="kernel_rbf" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+ <mxGeometry x="220" y="130" width="120" height="60" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-2" value="_logpdf" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+ <mxGeometry x="420" y="130" width="120" height="60" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-10" value="<p style="margin:0px;margin-top:4px;text-align:center;"><b>BayesInf</b></p><hr size="1"><div style="height:2px;"></div>" style="verticalAlign=top;align=left;overflow=fill;fontSize=12;fontFamily=Helvetica;html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="60" y="300" width="610" height="390" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-24" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-13">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-9" value="create_inference" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="340" y="340" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-25" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-14">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-27" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-13" value="perform_bootstrap" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="90" y="350" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-14" value="_perturb_data" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="90" y="540" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-15" value="_eval_model" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="500" y="610" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-16" value="normpdf" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="500" y="360" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-17" value="_corr_factor_BME" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="340" y="430" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-18" value="_rejection_sampling" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="160" y="420" width="120" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-26" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-19" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-19" value="_posterior_predictive" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="490" y="430" width="130" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-28" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-20" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-20" value="_plot_max_a_posteriori" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="240" y="610" width="140" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-21" value="plot_post_predictive" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="240" y="550" width="120" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-22" value="<p style="margin:0px;margin-top:4px;text-align:center;"><b>MCMC</b></p><hr size="1"><div style="height:2px;"></div>" style="verticalAlign=top;align=left;overflow=fill;fontSize=12;fontFamily=Helvetica;html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="170" y="750" width="140" height="60" as="geometry" />
+ </mxCell>
+ </root>
+ </mxGraphModel>
+ </diagram>
+</mxfile>
diff --git a/docs/diagrams/Structure_BayesInf.drawio b/docs/diagrams/Structure_BayesInf.drawio
new file mode 100644
index 0000000000000000000000000000000000000000..1651230d37cc9da9120a7f0ae60ff3450d8f225a
--- /dev/null
+++ b/docs/diagrams/Structure_BayesInf.drawio
@@ -0,0 +1,161 @@
+<mxfile host="Electron" modified="2024-04-10T11:36:50.177Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/22.1.11 Chrome/114.0.5735.289 Electron/25.9.8 Safari/537.36" etag="zH0uklPxig_FBrKRFuLP" version="22.1.11" type="device">
+ <diagram name="Page-1" id="efOe0Jku58RX-i1bv-3b">
+ <mxGraphModel dx="836" dy="1114" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100" math="0" shadow="0">
+ <root>
+ <mxCell id="0" />
+ <mxCell id="1" parent="0" />
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-1" value="_kernel_rbf" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+ <mxGeometry x="1070" y="200" width="120" height="60" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-2" value="_logpdf" style="rounded=0;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+ <mxGeometry x="860" y="130" width="120" height="60" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-10" value="<p style="margin:0px;margin-top:4px;text-align:center;"><b>BayesInf</b></p><hr size="1"><div style="height:2px;"></div>" style="verticalAlign=top;align=left;overflow=fill;fontSize=12;fontFamily=Helvetica;html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="40" y="280" width="1150" height="620" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-24" value="if self.bootstrap <br>or self.bayes_loocv <br>or self.just_analysis" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;labelBackgroundColor=#ffae00;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-13">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-31" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-18">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-42" value="if self.name != 'valid'<br>and self.inference_method != 'rejection'" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=default;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-31">
+ <mxGeometry x="0.5646" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-32" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-22">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-43" value="if self.inference_method == 'mcmc'" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-32">
+ <mxGeometry x="-0.0958" y="-1" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-33" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-19">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-52" value="always" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#C2C2C2;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-33">
+ <mxGeometry x="-0.112" y="1" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-34" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-21">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-47" value="if self.plot_post_pred" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-34">
+ <mxGeometry x="0.2399" y="-1" relative="1" as="geometry">
+ <mxPoint y="1" as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-35" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-9" target="xary-zVek9Bg-A1b1ZmA-20">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-46" value="if self.plot_map_pred" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-35">
+ <mxGeometry x="0.4183" y="-1" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-9" value="create_inference" style="html=1;whiteSpace=wrap;strokeWidth=2;" vertex="1" parent="1">
+ <mxGeometry x="405" y="539" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-25" value="if len(self.perturbed_data) == 0" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-14">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-27" value="if not self.emulator" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-29" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-16">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-44" value="always" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#cdcbcb;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-29">
+ <mxGeometry x="0.4722" y="1" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-30" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-13" target="xary-zVek9Bg-A1b1ZmA-17">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-41" value="if self.emulator" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-30">
+ <mxGeometry x="0.6143" y="-3" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-13" value="perform_bootstrap" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="150" y="310" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-14" value="_perturb_data" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="150" y="760" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-15" value="_eval_model" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="1050" y="660" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-38" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-16" target="xary-zVek9Bg-A1b1ZmA-1">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-49" value="if hasattr bias_inputs&nbsp;<br>and not hasattr error_model" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#ffae00;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-38">
+ <mxGeometry x="0.3126" y="-3" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-39" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-16" target="xary-zVek9Bg-A1b1ZmA-2">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-16" value="normpdf" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="650" y="390" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-40" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-17" target="xary-zVek9Bg-A1b1ZmA-2">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-50" value="always" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#cdcbcb;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-40">
+ <mxGeometry x="-0.6073" y="-5" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-17" value="_corr_factor_BME" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="650" y="450" width="110" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-18" value="_rejection_sampling" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="330" y="790" width="120" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-26" value="if not self.emulator&nbsp;<br>and not self.inference_method == 'rejection'&nbsp;<br>and self.name == 'calib" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-19" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-37" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-19" target="xary-zVek9Bg-A1b1ZmA-1">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-48" value="if sigma2_prior is not None<br>and if hasattr bias_inputs<br>and if not hasattr error_model" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#ffae00;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-37">
+ <mxGeometry x="-0.5544" y="-1" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-19" value="_posterior_predictive" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="675" y="590" width="130" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-28" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" edge="1" parent="1" source="xary-zVek9Bg-A1b1ZmA-20" target="xary-zVek9Bg-A1b1ZmA-15">
+ <mxGeometry relative="1" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-45" value="always" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];labelBackgroundColor=#cdcbcb;" vertex="1" connectable="0" parent="xary-zVek9Bg-A1b1ZmA-28">
+ <mxGeometry x="0.0517" relative="1" as="geometry">
+ <mxPoint as="offset" />
+ </mxGeometry>
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-20" value="_plot_max_a_posteriori" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="495" y="790" width="140" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-21" value="plot_post_predictive" style="html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="630" y="720" width="120" height="50" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-22" value="<p style="margin:0px;margin-top:4px;text-align:center;"><b>MCMC</b></p><hr size="1"><div style="height:2px;"></div>" style="verticalAlign=top;align=left;overflow=fill;fontSize=12;fontFamily=Helvetica;html=1;whiteSpace=wrap;" vertex="1" parent="1">
+ <mxGeometry x="1230" y="425" width="760" height="275" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-36" value="Note: Arrows indicate function calls, beginning calls the end" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+ <mxGeometry x="10" y="10" width="190" height="30" as="geometry" />
+ </mxCell>
+ <mxCell id="xary-zVek9Bg-A1b1ZmA-51" value="Color meanings:<br><span style="white-space: pre;">	</span>red: wrong, change<br><span style="white-space: pre;">	</span>orange: seems off, look at again" style="text;html=1;strokeColor=none;fillColor=none;align=left;verticalAlign=middle;whiteSpace=wrap;rounded=0;" vertex="1" parent="1">
+ <mxGeometry x="20" y="70" width="220" height="30" as="geometry" />
+ </mxCell>
+ </root>
+ </mxGraphModel>
+ </diagram>
+</mxfile>
diff --git a/examples/analytical-function/bayesvalidrox/__init__.py b/examples/analytical-function/bayesvalidrox/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e865af80652b8dd29203c2c85f8d1c717e335bc
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/__init__.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+__version__ = "0.0.5"
+
+from .pylink.pylink import PyLinkForwardModel
+from .surrogate_models.surrogate_models import MetaModel
+#from .surrogate_models.meta_model_engine import MetaModelEngine
+from .surrogate_models.engine import Engine
+from .surrogate_models.inputs import Input
+from .post_processing.post_processing import PostProcessing
+from .bayes_inference.bayes_inference import BayesInference
+from .bayes_inference.bayes_model_comparison import BayesModelComparison
+from .bayes_inference.discrepancy import Discrepancy
+
+__all__ = [
+ "__version__",
+ "PyLinkForwardModel",
+ "Input",
+ "Discrepancy",
+ "MetaModel",
+ #"MetaModelEngine",
+ "Engine",
+ "PostProcessing",
+ "BayesInference",
+ "BayesModelComparison"
+ ]
diff --git a/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..beaab3c798a63fcfbc361982388fdf10830a787e
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..afe0b0529fcde6ca9dbc06ed8932485f099ca476
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d5ebaedb4c8b77b5d7dbd0a6945f09079d8b10e4
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__init__.py b/examples/analytical-function/bayesvalidrox/bayes_inference/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..df8d935680f96ab487cf087866e8bfd504762945
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayes_inference/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+from .bayes_inference import BayesInference
+from .mcmc import MCMC
+
+__all__ = [
+ "BayesInference",
+ "MCMC"
+ ]
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..430b9885a8c8bd658da24bbc4ac1a6a0a74f69e6
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..72c63a98588c54dfec12536a99537cfa3a67e0cf
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..287257c1ca9b3f3a6d7e176e006ad432f8c685bf
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e89dfb5e6b3a873ac2f40dcc2084aa52caaedcd6
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..05467b4a003342c6353478050dd6b67db4347dcb
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a37330680c29bbfdba5a1bfd98041dda24957604
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..57322b839ff7d50ea32d3b36ce011c09cf91e232
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5f608d0ecf37288d6330349caabd5e7789533748
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b58f792ac36504c702205afc34228600f9bbba77
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bc71688a4cae4f74ec3a67838fca659881c520c9
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f505c78056d5629a52177e4fc2d67abdbd2aa48f
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f154800917aeaef9c499abf13d47fcca6ffc639b
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d2122246d54ec5697803cbce28900e077b2306a
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..245146ac5f5f72cb819fa504cdf288da3a5b75d3
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b9fe4689524895c2149048d489b22b08e85026df
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_inference.py b/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_inference.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ac43b62becf7441b2db90cf9b4ffeaab33c54bb
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_inference.py
@@ -0,0 +1,1530 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import os
+import copy
+import pandas as pd
+from tqdm import tqdm
+from scipy import stats
+import scipy.linalg as spla
+import joblib
+import seaborn as sns
+import corner
+import h5py
+import multiprocessing
+import gc
+from sklearn.metrics import mean_squared_error, r2_score
+from sklearn import preprocessing
+from matplotlib.patches import Patch
+import matplotlib.lines as mlines
+from matplotlib.backends.backend_pdf import PdfPages
+import matplotlib.pylab as plt
+
+from .mcmc import MCMC
+
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class BayesInference:
+ """
+ A class to perform Bayesian Analysis.
+
+
+ Attributes
+ ----------
+ MetaModel : obj
+ Meta model object.
+ discrepancy : obj
+ The discrepancy object for the sigma2s, i.e. the diagonal entries
+ of the variance matrix for a multivariate normal likelihood.
+ name : str, optional
+ The type of analysis, either calibration (`Calib`) or validation
+ (`Valid`). The default is `'Calib'`.
+ emulator : bool, optional
+ Analysis with emulator (MetaModel). The default is `True`.
+ bootstrap : bool, optional
+ Bootstrap the analysis. The default is `False`.
+ req_outputs : list, optional
+ The list of requested output to be used for the analysis.
+ The default is `None`. If None, all the defined outputs for the model
+ object is used.
+ selected_indices : dict, optional
+ A dictionary with the selected indices of each model output. The
+ default is `None`. If `None`, all measurement points are used in the
+ analysis.
+ samples : array of shape (n_samples, n_params), optional
+ The samples to be used in the analysis. The default is `None`. If
+ None the samples are drawn from the probablistic input parameter
+ object of the MetaModel object.
+ n_samples : int, optional
+ Number of samples to be used in the analysis. The default is `500000`.
+ If samples is not `None`, this argument will be assigned based on the
+ number of samples given.
+ measured_data : dict, optional
+ A dictionary containing the observation data. The default is `None`.
+ if `None`, the observation defined in the Model object of the
+ MetaModel is used.
+ inference_method : str, optional
+ A method for approximating the posterior distribution in the Bayesian
+ inference step. The default is `'rejection'`, which stands for
+ rejection sampling. A Markov Chain Monte Carlo sampler can be simply
+ selected by passing `'MCMC'`.
+ mcmc_params : dict, optional
+ A dictionary with args required for the Bayesian inference with
+ `MCMC`. The default is `None`.
+
+ Pass the mcmc_params like the following:
+
+ >>> mcmc_params:{
+ 'init_samples': None, # initial samples
+ 'n_walkers': 100, # number of walkers (chain)
+ 'n_steps': 100000, # number of maximum steps
+ 'n_burn': 200, # number of burn-in steps
+ 'moves': None, # Moves for the emcee sampler
+ 'multiprocessing': False, # multiprocessing
+ 'verbose': False # verbosity
+ }
+ The items shown above are the default values. If any parmeter is
+ not defined, the default value will be assigned to it.
+ bayes_loocv : bool, optional
+ Bayesian Leave-one-out Cross Validation. The default is `False`. If
+ `True`, the LOOCV procedure is used to estimate the bayesian Model
+ Evidence (BME).
+ n_bootstrap_itrs : int, optional
+ Number of bootstrap iteration. The default is `1`. If bayes_loocv is
+ `True`, this is qualt to the total length of the observation data
+ set.
+ perturbed_data : array of shape (n_bootstrap_itrs, n_obs), optional
+ User defined perturbed data. The default is `[]`.
+ bootstrap_noise : float, optional
+ A noise level to perturb the data set. The default is `0.05`.
+ just_analysis : bool, optional
+ Justifiability analysis. The default is False.
+ valid_metrics : list, optional
+ List of the validation metrics. The following metrics are supported:
+
+ 1. log_BME : logarithm of the Bayesian model evidence
+ 2. KLD : Kullback-Leibler Divergence
+ 3. inf_entropy: Information entropy
+ The default is `['log_BME']`.
+ plot_post_pred : bool, optional
+ Plot posterior predictive plots. The default is `True`.
+ plot_map_pred : bool, optional
+ Plot the model outputs vs the metamodel predictions for the maximum
+ a posteriori (defined as `max_a_posteriori`) parameter set. The
+ default is `False`.
+ max_a_posteriori : str, optional
+ Maximum a posteriori. `'mean'` and `'mode'` are available. The default
+ is `'mean'`.
+ corner_title_fmt : str, optional
+ Title format for the posterior distribution plot with python
+ package `corner`. The default is `'.2e'`.
+
+ """
+
+ def __init__(self, engine, MetaModel = None, discrepancy=None, emulator=True,
+ name='Calib', bootstrap=False, req_outputs=None,
+ selected_indices=None, samples=None, n_samples=100000,
+ measured_data=None, inference_method='rejection',
+ mcmc_params=None, bayes_loocv=False, n_bootstrap_itrs=1,
+ perturbed_data=[], bootstrap_noise=0.05, just_analysis=False,
+ valid_metrics=['BME'], plot_post_pred=True,
+ plot_map_pred=False, max_a_posteriori='mean',
+ corner_title_fmt='.2e'):
+
+ self.engine = engine
+ self.MetaModel = engine.MetaModel
+ self.Discrepancy = discrepancy
+ self.emulator = emulator
+ self.name = name
+ self.bootstrap = bootstrap
+ self.req_outputs = req_outputs
+ self.selected_indices = selected_indices
+ self.samples = samples
+ self.n_samples = n_samples
+ self.measured_data = measured_data
+ self.inference_method = inference_method
+ self.mcmc_params = mcmc_params
+ self.perturbed_data = perturbed_data
+ self.bayes_loocv = bayes_loocv
+ self.n_bootstrap_itrs = n_bootstrap_itrs
+ self.bootstrap_noise = bootstrap_noise
+ self.just_analysis = just_analysis
+ self.valid_metrics = valid_metrics
+ self.plot_post_pred = plot_post_pred
+ self.plot_map_pred = plot_map_pred
+ self.max_a_posteriori = max_a_posteriori
+ self.corner_title_fmt = corner_title_fmt
+
+ # -------------------------------------------------------------------------
+ def create_inference(self):
+ """
+ Starts the inference.
+
+ Returns
+ -------
+ BayesInference : obj
+ The Bayes inference object.
+
+ """
+
+ # Set some variables
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+ n_params = MetaModel.n_params
+ output_names = Model.Output.names
+ par_names = self.engine.ExpDesign.par_names
+
+ # If the prior is set by the user, take it.
+ if self.samples is None:
+ self.samples = self.engine.ExpDesign.generate_samples(
+ self.n_samples, 'random')
+ else:
+ try:
+ samples = self.samples.values
+ except AttributeError:
+ samples = self.samples
+
+ # Take care of an additional Sigma2s
+ self.samples = samples[:, :n_params]
+
+ # Update number of samples
+ self.n_samples = self.samples.shape[0]
+
+ # ---------- Preparation of observation data ----------
+ # Read observation data and perturb it if requested.
+ if self.measured_data is None:
+ self.measured_data = Model.read_observation(case=self.name)
+ # Convert measured_data to a data frame
+ if not isinstance(self.measured_data, pd.DataFrame):
+ self.measured_data = pd.DataFrame(self.measured_data)
+
+ # Extract the total number of measurement points
+ if self.name.lower() == 'calib':
+ self.n_tot_measurement = Model.n_obs
+ else:
+ self.n_tot_measurement = Model.n_obs_valid
+
+ # Find measurement error (if not given) for post predictive plot
+ if not hasattr(self, 'measurement_error'):
+ if isinstance(self.Discrepancy, dict):
+ Disc = self.Discrepancy['known']
+ else:
+ Disc = self.Discrepancy
+ if isinstance(Disc.parameters, dict):
+ self.measurement_error = {k: np.sqrt(Disc.parameters[k]) for k
+ in Disc.parameters.keys()}
+ else:
+ try:
+ self.measurement_error = np.sqrt(Disc.parameters)
+ except TypeError:
+ pass
+
+ # ---------- Preparation of variance for covariance matrix ----------
+ # Independent and identically distributed
+ total_sigma2 = dict()
+ opt_sigma_flag = isinstance(self.Discrepancy, dict)
+ opt_sigma = None
+ for key_idx, key in enumerate(output_names):
+ # Find opt_sigma
+ if opt_sigma_flag and opt_sigma is None:
+ # Option A: known error with unknown bias term
+ opt_sigma = 'A'
+ known_discrepancy = self.Discrepancy['known'] # TODO: the syntax here looks different from expected
+ self.Discrepancy = self.Discrepancy['infer'] # TODO: the syntax here looks different from expected
+ sigma2 = np.array(known_discrepancy.parameters[key])
+
+ elif self.Discrepancy.parameters is not None:
+ # Option B: The sigma2 is known (no bias term)
+ opt_sigma = 'B'
+ sigma2 = np.array(self.Discrepancy.parameters[key])
+
+ elif not isinstance(self.Discrepancy.InputDisc, str):
+ # Option C: The sigma2 is unknown (bias term including error)
+ opt_sigma = 'C'
+ n_measurement = self.measured_data[key].values.shape
+ sigma2 = np.zeros((n_measurement[0]))
+
+ total_sigma2[key] = sigma2
+
+ self.Discrepancy.opt_sigma = opt_sigma
+ self.Discrepancy.total_sigma2 = total_sigma2
+
+ # If inferred sigma2s obtained from e.g. calibration are given
+ try:
+ self.sigma2s = self.Discrepancy.get_sample(self.n_samples)
+ except:
+ pass #TODO: should an error be raised in this case?
+
+ # ---------------- Bootstrap & TOM --------------------
+ if self.bootstrap or self.bayes_loocv or self.just_analysis:
+ if len(self.perturbed_data) == 0:
+ # zero mean noise Adding some noise to the observation function
+ self.perturbed_data = self._perturb_data(
+ self.measured_data, output_names
+ )
+ else:
+ self.n_bootstrap_itrs = len(self.perturbed_data)
+
+ # -------- Model Discrepancy -----------
+ if hasattr(self, 'error_model') and self.error_model \
+ and self.name.lower() != 'calib':
+ # Select posterior mean as MAP
+ MAP_theta = self.samples.mean(axis=0).reshape((1, n_params))
+ # MAP_theta = stats.mode(self.samples,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=MAP_theta)
+
+ # Train a GPR meta-model using MAP
+ self.error_MetaModel = MetaModel.create_model_error(
+ self.bias_inputs, y_MAP, Name=self.name
+ )
+
+ # -----------------------------------------------------
+ # ----- Loop over the perturbed observation data ------
+ # -----------------------------------------------------
+ # Initilize arrays
+ logLikelihoods = np.zeros((self.n_samples, self.n_bootstrap_itrs),
+ dtype=np.float16)
+ BME_Corr = np.zeros((self.n_bootstrap_itrs))
+ log_BME = np.zeros((self.n_bootstrap_itrs))
+ KLD = np.zeros((self.n_bootstrap_itrs))
+ inf_entropy = np.zeros((self.n_bootstrap_itrs))
+
+ # Compute the prior predtions
+ # Evaluate the MetaModel
+ if self.emulator:
+ y_hat, y_std = MetaModel.eval_metamodel(samples=self.samples)
+ self.__mean_pce_prior_pred = y_hat
+ self._std_pce_prior_pred = y_std
+
+ # Correct the predictions with Model discrepancy
+ if hasattr(self, 'error_model') and self.error_model:
+ y_hat_corr, y_std = self.error_MetaModel.eval_model_error(
+ self.bias_inputs, self.__mean_pce_prior_pred
+ )
+ self.__mean_pce_prior_pred = y_hat_corr
+ self._std_pce_prior_pred = y_std
+
+ # Surrogate model's error using RMSE of test data
+ if hasattr(MetaModel, 'rmse'):
+ surrError = MetaModel.rmse
+ else:
+ surrError = None
+
+ else:
+ # Evaluate the original model
+ self.__model_prior_pred = self._eval_model(
+ samples=self.samples, key='PriorPred'
+ )
+ surrError = None
+
+ # Start the likelihood-BME computations for the perturbed data
+ for itr_idx, data in tqdm(
+ enumerate(self.perturbed_data),
+ total=self.n_bootstrap_itrs,
+ desc="Bootstrapping the BME calculations", ascii=True
+ ):
+
+ # ---------------- Likelihood calculation ----------------
+ if self.emulator:
+ model_evals = self.__mean_pce_prior_pred
+ else:
+ model_evals = self.__model_prior_pred
+
+ # Leave one out
+ if self.bayes_loocv or self.just_analysis:
+ self.selected_indices = np.nonzero(data)[0]
+
+ # Prepare data dataframe
+ nobs = list(self.measured_data.count().values[1:])
+ numbers = list(np.cumsum(nobs))
+ indices = list(zip([0] + numbers, numbers))
+ data_dict = {
+ output_names[i]: data[j:k] for i, (j, k) in
+ enumerate(indices)
+ }
+ #print(output_names)
+ #print(indices)
+ #print(numbers)
+ #print(nobs)
+ #print(self.measured_data)
+ #for i, (j, k) in enumerate(indices):
+ # print(i,j,k)
+ #print(data)
+ #print(data_dict)
+ #stop
+
+ # Unknown sigma2
+ if opt_sigma == 'C' or hasattr(self, 'sigma2s'):
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ sigma2=self.sigma2s, std=surrError
+ )
+ else:
+ # known sigma2
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ std=surrError
+ )
+ # ---------------- BME Calculations ----------------
+ # BME (log)
+ log_BME[itr_idx] = np.log(
+ np.nanmean(np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.longdouble))#float128))
+ )
+
+ # BME correction when using Emulator
+ if self.emulator:
+ BME_Corr[itr_idx] = self.__corr_factor_BME(
+ data_dict, total_sigma2, log_BME[itr_idx]
+ )
+
+ # Rejection Step
+ if 'kld' in list(map(str.lower, self.valid_metrics)) and\
+ 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, self.n_samples)[0]
+
+ # Reject the poorly performed prior
+ Likelihoods = np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.float64)
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ posterior = self.samples[accepted]
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(
+ logLikelihoods[:, itr_idx][accepted]
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ KLD[itr_idx] = postExpLikelihoods - log_BME[itr_idx]
+
+ # Posterior-based expectation of prior densities
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ n_thread = int(0.875 * multiprocessing.cpu_count())
+ with multiprocessing.Pool(n_thread) as p:
+ postExpPrior = np.mean(np.concatenate(
+ p.map(
+ self.engine.ExpDesign.JDist.pdf,
+ np.array_split(posterior.T, n_thread, axis=1))
+ )
+ )
+ # Information Entropy based on Entropy paper Eq. 38
+ inf_entropy[itr_idx] = log_BME[itr_idx] - postExpPrior - \
+ postExpLikelihoods
+
+ # Clear memory
+ gc.collect(generation=2)
+
+ # ---------- Store metrics for perturbed data set ----------------
+ # Likelihoods (Size: n_samples, n_bootstrap_itr)
+ self.log_likes = logLikelihoods
+
+ # BME (log), KLD, infEntropy (Size: 1,n_bootstrap_itr)
+ self.log_BME = log_BME
+
+ # BMECorrFactor (log) (Size: 1,n_bootstrap_itr)
+ if self.emulator:
+ self.log_BME_corr_factor = BME_Corr
+
+ if 'kld' in list(map(str.lower, self.valid_metrics)):
+ self.KLD = KLD
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ self.inf_entropy = inf_entropy
+
+ # BME = BME + BMECorrFactor
+ if self.emulator:
+ self.log_BME += self.log_BME_corr_factor
+
+ # ---------------- Parameter Bayesian inference ----------------
+ if self.inference_method.lower() == 'mcmc':
+ # Instantiate the MCMC object
+ MCMC_Obj = MCMC(self)
+ self.posterior_df = MCMC_Obj.run_sampler(
+ self.measured_data, total_sigma2
+ )
+
+ elif self.name.lower() == 'valid':
+ # Convert to a dataframe if samples are provided after calibration.
+ self.posterior_df = pd.DataFrame(self.samples, columns=par_names)
+
+ else:
+ # Rejection sampling
+ self.posterior_df = self._rejection_sampling()
+
+ # Provide posterior's summary
+ print('\n')
+ print('-'*15 + 'Posterior summary' + '-'*15)
+ pd.options.display.max_columns = None
+ pd.options.display.max_rows = None
+ print(self.posterior_df.describe())
+ print('-'*50)
+
+ # -------- Model Discrepancy -----------
+ if hasattr(self, 'error_model') and self.error_model \
+ and self.name.lower() == 'calib':
+ if self.inference_method.lower() == 'mcmc':
+ self.error_MetaModel = MCMC_Obj.error_MetaModel
+ else:
+ # Select posterior mean as MAP
+ if opt_sigma == "B":
+ posterior_df = self.posterior_df.values
+ else:
+ posterior_df = self.posterior_df.values[:, :-Model.n_outputs]
+
+ # Select posterior mean as Maximum a posteriori
+ map_theta = posterior_df.mean(axis=0).reshape((1, n_params))
+ # map_theta = stats.mode(Posterior_df,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=map_theta)
+
+ # Train a GPR meta-model using MAP
+ self.error_MetaModel = MetaModel.create_model_error(
+ self.bias_inputs, y_MAP, Name=self.name
+ )
+
+ # -------- Posterior perdictive -----------
+ self._posterior_predictive()
+
+ # -----------------------------------------------------
+ # ------------------ Visualization --------------------
+ # -----------------------------------------------------
+ # Create Output directory, if it doesn't exist already.
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ os.makedirs(out_dir, exist_ok=True)
+
+ # -------- Posteior parameters --------
+ if opt_sigma != "B":
+ par_names.extend(
+ [self.Discrepancy.InputDisc.Marginals[i].name for i
+ in range(len(self.Discrepancy.InputDisc.Marginals))]
+ )
+ # Pot with corner
+ figPosterior = corner.corner(self.posterior_df.to_numpy(),
+ labels=par_names,
+ quantiles=[0.15, 0.5, 0.85],
+ show_titles=True,
+ title_fmt=self.corner_title_fmt,
+ labelpad=0.2,
+ use_math_text=True,
+ title_kwargs={"fontsize": 28},
+ plot_datapoints=False,
+ plot_density=False,
+ fill_contours=True,
+ smooth=0.5,
+ smooth1d=0.5)
+
+ # Loop over axes and set x limits
+ if opt_sigma == "B":
+ axes = np.array(figPosterior.axes).reshape(
+ (len(par_names), len(par_names))
+ )
+ for yi in range(len(par_names)):
+ ax = axes[yi, yi]
+ ax.set_xlim(self.engine.ExpDesign.bound_tuples[yi])
+ for xi in range(yi):
+ ax = axes[yi, xi]
+ ax.set_xlim(self.engine.ExpDesign.bound_tuples[xi])
+ plt.close()
+
+ # Turn off gridlines
+ for ax in figPosterior.axes:
+ ax.grid(False)
+
+ if self.emulator:
+ plotname = f'/Posterior_Dist_{Model.name}_emulator'
+ else:
+ plotname = f'/Posterior_Dist_{Model.name}'
+
+ figPosterior.set_size_inches((24, 16))
+ figPosterior.savefig(f'./{out_dir}{plotname}.pdf',
+ bbox_inches='tight')
+
+ # -------- Plot MAP --------
+ if self.plot_map_pred:
+ self._plot_max_a_posteriori()
+
+ # -------- Plot log_BME dist --------
+ if self.bootstrap:
+
+ # Computing the TOM performance
+ self.log_BME_tom = stats.chi2.rvs(
+ self.n_tot_measurement, size=self.log_BME.shape[0]
+ )
+
+ fig, ax = plt.subplots()
+ sns.kdeplot(self.log_BME_tom, ax=ax, color="green", shade=True)
+ sns.kdeplot(
+ self.log_BME, ax=ax, color="blue", shade=True,
+ label='Model BME')
+
+ ax.set_xlabel('log$_{10}$(BME)')
+ ax.set_ylabel('Probability density')
+
+ legend_elements = [
+ Patch(facecolor='green', edgecolor='green', label='TOM BME'),
+ Patch(facecolor='blue', edgecolor='blue', label='Model BME')
+ ]
+ ax.legend(handles=legend_elements)
+
+ if self.emulator:
+ plotname = f'/BME_hist_{Model.name}_emulator'
+ else:
+ plotname = f'/BME_hist_{Model.name}'
+
+ plt.savefig(f'./{out_dir}{plotname}.pdf', bbox_inches='tight')
+ plt.show()
+ plt.close()
+
+ # -------- Posteior perdictives --------
+ if self.plot_post_pred:
+ # Plot the posterior predictive
+ self._plot_post_predictive()
+
+ return self
+
+ # -------------------------------------------------------------------------
+ def _perturb_data(self, data, output_names):
+ """
+ Returns an array with n_bootstrap_itrs rowsof perturbed data.
+ The first row includes the original observation data.
+ If `self.bayes_loocv` is True, a 2d-array will be returned with
+ repeated rows and zero diagonal entries.
+
+ Parameters
+ ----------
+ data : pandas DataFrame
+ Observation data.
+ output_names : list
+ List of the output names.
+
+ Returns
+ -------
+ final_data : array
+ Perturbed data set.
+
+ """
+ noise_level = self.bootstrap_noise
+ obs_data = data[output_names].values
+ n_measurement, n_outs = obs_data.shape
+ self.n_tot_measurement = obs_data[~np.isnan(obs_data)].shape[0]
+ # Number of bootstrap iterations
+ if self.bayes_loocv:
+ self.n_bootstrap_itrs = self.n_tot_measurement
+
+ # Pass loocv dataset
+ if self.bayes_loocv:
+ obs = obs_data.T[~np.isnan(obs_data.T)]
+ final_data = np.repeat(np.atleast_2d(obs), self.n_bootstrap_itrs,
+ axis=0)
+ np.fill_diagonal(final_data, 0)
+ return final_data
+
+ else:
+ final_data = np.zeros(
+ (self.n_bootstrap_itrs, self.n_tot_measurement)
+ )
+ final_data[0] = obs_data.T[~np.isnan(obs_data.T)]
+ for itrIdx in range(1, self.n_bootstrap_itrs):
+ data = np.zeros((n_measurement, n_outs))
+ for idx in range(len(output_names)):
+ std = np.nanstd(obs_data[:, idx])
+ if std == 0:
+ std = 0.001
+ noise = std * noise_level
+ data[:, idx] = np.add(
+ obs_data[:, idx],
+ np.random.normal(0, 1, obs_data.shape[0]) * noise,
+ )
+
+ final_data[itrIdx] = data.T[~np.isnan(data.T)]
+
+ return final_data
+
+ # -------------------------------------------------------------------------
+ def _logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = spla.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(spla.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def _eval_model(self, samples=None, key='MAP'):
+ """
+ Evaluates Forward Model.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets. The default is None.
+ key : str, optional
+ Key string to be passed to the run_model_parallel method.
+ The default is 'MAP'.
+
+ Returns
+ -------
+ model_outputs : dict
+ Model outputs.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+
+ if samples is None:
+ self.samples = self.engine.ExpDesign.generate_samples(
+ self.n_samples, 'random')
+ else:
+ self.samples = samples
+ self.n_samples = len(samples)
+
+ model_outputs, _ = Model.run_model_parallel(
+ self.samples, key_str=key+self.name)
+
+ # Clean up
+ # Zip the subdirectories
+ try:
+ dir_name = f'{Model.name}MAP{self.name}'
+ key = dir_name + '_'
+ Model.zip_subdirs(dir_name, key)
+ except:
+ pass
+
+ return model_outputs
+
+ # -------------------------------------------------------------------------
+ def _kernel_rbf(self, X, hyperparameters):
+ """
+ Isotropic squared exponential kernel.
+
+ Higher l values lead to smoother functions and therefore to coarser
+ approximations of the training data. Lower l values make functions
+ more wiggly with wide uncertainty regions between training data points.
+
+ sigma_f controls the marginal variance of b(x)
+
+ Parameters
+ ----------
+ X : ndarray of shape (n_samples_X, n_features)
+
+ hyperparameters : Dict
+ Lambda characteristic length
+ sigma_f controls the marginal variance of b(x)
+ sigma_0 unresolvable error nugget term, interpreted as random
+ error that cannot be attributed to measurement error.
+ Returns
+ -------
+ var_cov_matrix : ndarray of shape (n_samples_X,n_samples_X)
+ Kernel k(X, X).
+
+ """
+ from sklearn.gaussian_process.kernels import RBF
+ min_max_scaler = preprocessing.MinMaxScaler()
+ X_minmax = min_max_scaler.fit_transform(X)
+
+ nparams = len(hyperparameters)
+ # characteristic length (0,1]
+ Lambda = hyperparameters[0]
+ # sigma_f controls the marginal variance of b(x)
+ sigma2_f = hyperparameters[1]
+
+ # cov_matrix = sigma2_f*rbf_kernel(X_minmax, gamma = 1/Lambda**2)
+
+ rbf = RBF(length_scale=Lambda)
+ cov_matrix = sigma2_f * rbf(X_minmax)
+ if nparams > 2:
+ # (unresolvable error) nugget term that is interpreted as random
+ # error that cannot be attributed to measurement error.
+ sigma2_0 = hyperparameters[2:]
+ for i, j in np.ndindex(cov_matrix.shape):
+ cov_matrix[i, j] += np.sum(sigma2_0) if i == j else 0
+
+ return cov_matrix
+
+ # -------------------------------------------------------------------------
+ def normpdf(self, outputs, obs_data, total_sigma2s, sigma2=None, std=None):
+ """
+ Calculates the likelihood of simulation outputs compared with
+ observation data.
+
+ Parameters
+ ----------
+ outputs : dict
+ A dictionary containing the simulation outputs as array of shape
+ (n_samples, n_measurement) for each model output.
+ obs_data : dict
+ A dictionary/dataframe containing the observation data.
+ total_sigma2s : dict
+ A dictionary with known values of the covariance diagonal entries,
+ a.k.a sigma^2.
+ sigma2 : array, optional
+ An array of the sigma^2 samples, when the covariance diagonal
+ entries are unknown and are being jointly inferred. The default is
+ None.
+ std : dict, optional
+ A dictionary containing the root mean squared error as array of
+ shape (n_samples, n_measurement) for each model output. The default
+ is None.
+
+ Returns
+ -------
+ logLik : array of shape (n_samples)
+ Likelihoods.
+
+ """
+ Model = self.engine.Model
+ logLik = 0.0
+
+ # Extract the requested model outputs for likelihood calulation
+ if self.req_outputs is None:
+ req_outputs = Model.Output.names
+ else:
+ req_outputs = list(self.req_outputs)
+
+ # Loop over the outputs
+ for idx, out in enumerate(req_outputs):
+
+ # (Meta)Model Output
+ nsamples, nout = outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Add the std of the PCE is chosen as emulator.
+ if self.emulator:
+ if std is not None:
+ tot_sigma2s += std[out]**2
+
+ # Covariance Matrix
+ covMatrix = np.diag(tot_sigma2s)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+
+ # If sigma2 is not given, use given total_sigma2s
+ if sigma2 is None:
+ logLik += stats.multivariate_normal.logpdf(
+ outputs[out][:, indices], data[indices], covMatrix)
+ #print('In if')
+ #print(logLik)
+ continue
+
+ # Loop over each run/sample and calculate logLikelihood
+ logliks = np.zeros(nsamples)
+ for s_idx in range(nsamples):
+
+ # Simulation run
+ tot_outputs = outputs[out]
+
+ # Covariance Matrix
+ covMatrix = np.diag(tot_sigma2s)
+
+ if sigma2 is not None:
+ # Check the type error term
+ if hasattr(self, 'bias_inputs') and \
+ not hasattr(self, 'error_model'):
+ # Infer a Bias model usig Gaussian Process Regression
+ bias_inputs = np.hstack(
+ (self.bias_inputs[out],
+ tot_outputs[s_idx].reshape(-1, 1)))
+
+ params = sigma2[s_idx, idx*3:(idx+1)*3]
+ covMatrix = self._kernel_rbf(bias_inputs, params)
+ else:
+ # Infer equal sigma2s
+ try:
+ sigma_2 = sigma2[s_idx, idx]
+ except TypeError:
+ sigma_2 = 0.0
+
+ covMatrix += sigma_2 * np.eye(nout)
+ # covMatrix = np.diag(sigma2 * total_sigma2s)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+
+ # Compute loglikelihood
+ logliks[s_idx] = self._logpdf(
+ tot_outputs[s_idx, indices], data[indices], covMatrix
+ )
+ #print('Continued')
+ #print(logliks)
+ logLik += logliks
+ #print(logLik)
+ return logLik
+
+ # -------------------------------------------------------------------------
+ def _corr_factor_BME_old(self, Data, total_sigma2s, posterior):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ OrigModelOutput = self.engine.ExpDesign.Y
+ Model = self.engine.Model
+
+ # Posterior with guassian-likelihood
+ postDist = stats.gaussian_kde(posterior.T)
+
+ # Remove NaN
+ Data = Data[~np.isnan(Data)]
+ total_sigma2s = total_sigma2s[~np.isnan(total_sigma2s)]
+
+ # Covariance Matrix
+ covMatrix = np.diag(total_sigma2s[:self.n_tot_measurement])
+
+ # Extract the requested model outputs for likelihood calulation
+ if self.req_outputs is None:
+ OutputType = Model.Output.names
+ else:
+ OutputType = list(self.req_outputs)
+
+ # SampleSize = OrigModelOutput[OutputType[0]].shape[0]
+
+
+ # Flatten the OutputType for OrigModel
+ TotalOutputs = np.concatenate([OrigModelOutput[x] for x in OutputType], 1)
+
+ NrofBayesSamples = self.n_samples
+ # Evaluate MetaModel on the experimental design
+ Samples = self.engine.ExpDesign.X
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=Samples)
+
+ # Reset the NrofSamples to NrofBayesSamples
+ self.n_samples = NrofBayesSamples
+
+ # Flatten the OutputType for MetaModel
+ TotalPCEOutputs = np.concatenate([OutputRS[x] for x in OutputRS], 1)
+ TotalPCEstdOutputRS= np.concatenate([stdOutputRS[x] for x in stdOutputRS], 1)
+
+ logweight = 0
+ for i, sample in enumerate(Samples):
+ # Compute likelilhood output vs RS
+ covMatrix = np.diag(TotalPCEstdOutputRS[i]**2)
+ logLik = self._logpdf(TotalOutputs[i], TotalPCEOutputs[i], covMatrix)
+ # Compute posterior likelihood of the collocation points
+ logpostLik = np.log(postDist.pdf(sample[:, None]))[0]
+ if logpostLik != -np.inf:
+ logweight += logLik + logpostLik
+ return logweight
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = self.engine.ExpDesign.X
+ model_outputs = self.engine.ExpDesign.Y
+ Model = self.engine.Model
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ covMatrix = np.eye(len(y_m)) * 1/(2*np.pi)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+ covMatrix_data = np.diag(covMatrix_data[indices, indices])
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self._logpdf(
+ y_m_hat[indices], data[indices],
+ covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self._logpdf(
+ y_m_hat[indices], y_m[indices],
+ covMatrix
+ )
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.mean(np.exp(logLik_model+logLik_data))
+
+ return np.log(weights)
+
+ # -------------------------------------------------------------------------
+ def _rejection_sampling(self):
+ """
+ Performs rejection sampling to update the prior distribution on the
+ input parameters.
+
+ Returns
+ -------
+ posterior : pandas.dataframe
+ Posterior samples of the input parameters.
+
+ """
+
+ MetaModel = self.MetaModel
+ try:
+ sigma2_prior = self.Discrepancy.sigma2_prior
+ except:
+ sigma2_prior = None
+
+ # Check if the discrepancy is defined as a distribution:
+ samples = self.samples
+
+ if sigma2_prior is not None:
+ samples = np.hstack((samples, sigma2_prior))
+
+ # Take the first column of Likelihoods (Observation data without noise)
+ if self.just_analysis or self.bayes_loocv:
+ index = self.n_tot_measurement-1
+ likelihoods = np.exp(self.log_likes[:, index], dtype=np.longdouble)#np.float128)
+ else:
+ likelihoods = np.exp(self.log_likes[:, 0], dtype=np.longdouble)#np.float128)
+
+ n_samples = len(likelihoods)
+ norm_ikelihoods = likelihoods / np.max(likelihoods)
+
+ # Normalize based on min if all Likelihoods are zero
+ if all(likelihoods == 0.0):
+ likelihoods = self.log_likes[:, 0]
+ norm_ikelihoods = likelihoods / np.min(likelihoods)
+
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, n_samples)[0]
+
+ # Reject the poorly performed prior
+ accepted_samples = samples[norm_ikelihoods >= unif]
+
+ # Output the Posterior
+ par_names = self.engine.ExpDesign.par_names
+ if sigma2_prior is not None:
+ for name in self.Discrepancy.name:
+ par_names.append(name)
+
+ return pd.DataFrame(accepted_samples, columns=sigma2_prior)
+
+ # -------------------------------------------------------------------------
+ def _posterior_predictive(self):
+ """
+ Stores the prior- and posterior predictive samples, i.e. model
+ evaluations using the samples, into hdf5 files.
+
+ priorPredictive.hdf5 : Prior predictive samples.
+ postPredictive_wo_noise.hdf5 : Posterior predictive samples without
+ the additive noise.
+ postPredictive.hdf5 : Posterior predictive samples with the additive
+ noise.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+
+ # Make a directory to save the prior/posterior predictive
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ os.makedirs(out_dir, exist_ok=True)
+
+ # Read observation data and perturb it if requested
+ if self.measured_data is None:
+ self.measured_data = Model.read_observation(case=self.name)
+
+ if not isinstance(self.measured_data, pd.DataFrame):
+ self.measured_data = pd.DataFrame(self.measured_data)
+
+ # X_values
+ x_values = self.engine.ExpDesign.x_values
+
+ try:
+ sigma2_prior = self.Discrepancy.sigma2_prior
+ except:
+ sigma2_prior = None
+
+ # Extract posterior samples
+ posterior_df = self.posterior_df
+
+ # Take care of the sigma2
+ if sigma2_prior is not None:
+ try:
+ sigma2s = posterior_df[self.Discrepancy.name].values
+ posterior_df = posterior_df.drop(
+ labels=self.Discrepancy.name, axis=1
+ )
+ except:
+ sigma2s = self.sigma2s
+
+ # Posterior predictive
+ if self.emulator:
+ if self.inference_method == 'rejection':
+ prior_pred = self.__mean_pce_prior_pred
+ if self.name.lower() != 'calib':
+ post_pred = self.__mean_pce_prior_pred
+ post_pred_std = self._std_pce_prior_pred
+ else:
+ post_pred, post_pred_std = MetaModel.eval_metamodel(
+ samples=posterior_df.values
+ )
+
+ else:
+ if self.inference_method == 'rejection':
+ prior_pred = self.__model_prior_pred
+ if self.name.lower() != 'calib':
+ post_pred = self.__mean_pce_prior_pred,
+ post_pred_std = self._std_pce_prior_pred
+ else:
+ post_pred = self._eval_model(
+ samples=posterior_df.values, key='PostPred'
+ )
+ # Correct the predictions with Model discrepancy
+ if hasattr(self, 'error_model') and self.error_model:
+ y_hat, y_std = self.error_MetaModel.eval_model_error(
+ self.bias_inputs, post_pred
+ )
+ post_pred, post_pred_std = y_hat, y_std
+
+ # Add discrepancy from likelihood Sample to the current posterior runs
+ total_sigma2 = self.Discrepancy.total_sigma2
+ post_pred_withnoise = copy.deepcopy(post_pred)
+ for varIdx, var in enumerate(Model.Output.names):
+ for i in range(len(post_pred[var])):
+ pred = post_pred[var][i]
+
+ # Known sigma2s
+ clean_sigma2 = total_sigma2[var][~np.isnan(total_sigma2[var])]
+ tot_sigma2 = clean_sigma2[:len(pred)]
+ cov = np.diag(tot_sigma2)
+
+ # Check the type error term
+ if sigma2_prior is not None:
+ # Inferred sigma2s
+ if hasattr(self, 'bias_inputs') and \
+ not hasattr(self, 'error_model'):
+ # TODO: Infer a Bias model usig GPR
+ bias_inputs = np.hstack((
+ self.bias_inputs[var], pred.reshape(-1, 1)))
+ params = sigma2s[i, varIdx*3:(varIdx+1)*3]
+ cov = self._kernel_rbf(bias_inputs, params)
+ else:
+ # Infer equal sigma2s
+ try:
+ sigma2 = sigma2s[i, varIdx]
+ except TypeError:
+ sigma2 = 0.0
+
+ # Convert biasSigma2s to a covMatrix
+ cov += sigma2 * np.eye(len(pred))
+
+ if self.emulator:
+ if hasattr(MetaModel, 'rmse') and \
+ MetaModel.rmse is not None:
+ stdPCE = MetaModel.rmse[var]
+ else:
+ stdPCE = post_pred_std[var][i]
+ # Expected value of variance (Assump: i.i.d stds)
+ cov += np.diag(stdPCE**2)
+
+ # Sample a multivariate normal distribution with mean of
+ # prediction and variance of cov
+ post_pred_withnoise[var][i] = np.random.multivariate_normal(
+ pred, cov, 1
+ )
+
+ # ----- Prior Predictive -----
+ if self.inference_method.lower() == 'rejection':
+ # Create hdf5 metadata
+ hdf5file = f'{out_dir}/priorPredictive.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=prior_pred[var])
+
+ # ----- Posterior Predictive only model evaluations -----
+ # Create hdf5 metadata
+ hdf5file = out_dir+'/postPredictive_wo_noise.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=post_pred[var])
+
+ # ----- Posterior Predictive with noise -----
+ # Create hdf5 metadata
+ hdf5file = out_dir+'/postPredictive.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=post_pred_withnoise[var])
+
+ return
+
+ # -------------------------------------------------------------------------
+ def _plot_max_a_posteriori(self):
+ """
+ Plots the response of the model output against that of the metamodel at
+ the maximum a posteriori sample (mean or mode of posterior.)
+
+ Returns
+ -------
+ None.
+
+ """
+
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ opt_sigma = self.Discrepancy.opt_sigma
+
+ # -------- Find MAP and run MetaModel and origModel --------
+ # Compute the MAP
+ if self.max_a_posteriori.lower() == 'mean':
+ if opt_sigma == "B":
+ Posterior_df = self.posterior_df.values
+ else:
+ Posterior_df = self.posterior_df.values[:, :-Model.n_outputs]
+ map_theta = Posterior_df.mean(axis=0).reshape(
+ (1, MetaModel.n_params))
+ else:
+ map_theta = stats.mode(Posterior_df.values, axis=0)[0]
+ # Prin report
+ print("\nPoint estimator:\n", map_theta[0])
+
+ # Run the models for MAP
+ # MetaModel
+ map_metamodel_mean, map_metamodel_std = MetaModel.eval_metamodel(
+ samples=map_theta)
+ self.map_metamodel_mean = map_metamodel_mean
+ self.map_metamodel_std = map_metamodel_std
+
+ # origModel
+ map_orig_model = self._eval_model(samples=map_theta)
+ self.map_orig_model = map_orig_model
+
+ # Extract slicing index
+ x_values = map_orig_model['x_values']
+
+ # List of markers and colors
+ Color = ['k', 'b', 'g', 'r']
+ Marker = 'x'
+
+ # Create a PdfPages object
+ pdf = PdfPages(f'./{out_dir}MAP_PCE_vs_Model_{self.name}.pdf')
+ fig = plt.figure()
+ for i, key in enumerate(Model.Output.names):
+
+ y_val = map_orig_model[key]
+ y_pce_val = map_metamodel_mean[key]
+ y_pce_val_std = map_metamodel_std[key]
+
+ plt.plot(x_values, y_val, color=Color[i], marker=Marker,
+ lw=2.0, label='$Y_{MAP}^{M}$')
+
+ plt.plot(
+ x_values, y_pce_val[i], color=Color[i], lw=2.0,
+ marker=Marker, linestyle='--', label='$Y_{MAP}^{PCE}$'
+ )
+ # plot the confidence interval
+ plt.fill_between(
+ x_values, y_pce_val[i] - 1.96*y_pce_val_std[i],
+ y_pce_val[i] + 1.96*y_pce_val_std[i],
+ color=Color[i], alpha=0.15
+ )
+
+ # Calculate the adjusted R_squared and RMSE
+ R2 = r2_score(y_pce_val.reshape(-1, 1), y_val.reshape(-1, 1))
+ rmse = np.sqrt(mean_squared_error(y_pce_val, y_val))
+
+ plt.ylabel(key)
+ plt.xlabel("Time [s]")
+ plt.title(f'Model vs MetaModel {key}')
+
+ ax = fig.axes[0]
+ leg = ax.legend(loc='best', frameon=True)
+ fig.canvas.draw()
+ p = leg.get_window_extent().inverse_transformed(ax.transAxes)
+ ax.text(
+ p.p0[1]-0.05, p.p1[1]-0.25,
+ f'RMSE = {rmse:.3f}\n$R^2$ = {R2:.3f}',
+ transform=ax.transAxes, color='black',
+ bbox=dict(facecolor='none', edgecolor='black',
+ boxstyle='round,pad=1'))
+
+ plt.show()
+
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ pdf.close()
+
+ # -------------------------------------------------------------------------
+ def _plot_post_predictive(self):
+ """
+ Plots the posterior predictives against the observation data.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ Model = self.engine.Model
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ # Plot the posterior predictive
+ for out_idx, out_name in enumerate(Model.Output.names):
+ fig, ax = plt.subplots()
+ with sns.axes_style("ticks"):
+ x_key = list(self.measured_data)[0]
+
+ # --- Read prior and posterior predictive ---
+ if self.inference_method == 'rejection' and \
+ self.name.lower() != 'valid':
+ # --- Prior ---
+ # Load posterior predictive
+ f = h5py.File(
+ f'{out_dir}/priorPredictive.hdf5', 'r+')
+
+ try:
+ x_coords = np.array(f[f"x_values/{out_name}"])
+ except:
+ x_coords = np.array(f["x_values"])
+
+ X_values = np.repeat(x_coords, 10000)
+
+ prior_pred_df = {}
+ prior_pred_df[x_key] = X_values
+ prior_pred_df[out_name] = np.array(
+ f[f"EDY/{out_name}"])[:10000].flatten('F')
+ prior_pred_df = pd.DataFrame(prior_pred_df)
+
+ tags_post = ['prior'] * len(prior_pred_df)
+ prior_pred_df.insert(
+ len(prior_pred_df.columns), "Tags", tags_post,
+ True)
+ f.close()
+
+ # --- Posterior ---
+ f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+
+ X_values = np.repeat(
+ x_coords, np.array(f[f"EDY/{out_name}"]).shape[0])
+
+ post_pred_df = {}
+ post_pred_df[x_key] = X_values
+ post_pred_df[out_name] = np.array(
+ f[f"EDY/{out_name}"]).flatten('F')
+
+ post_pred_df = pd.DataFrame(post_pred_df)
+
+ tags_post = ['posterior'] * len(post_pred_df)
+ post_pred_df.insert(
+ len(post_pred_df.columns), "Tags", tags_post, True)
+ f.close()
+ # Concatenate two dataframes based on x_values
+ frames = [prior_pred_df, post_pred_df]
+ all_pred_df = pd.concat(frames)
+
+ # --- Plot posterior predictive ---
+ sns.violinplot(
+ x_key, y=out_name, data=all_pred_df, hue="Tags",
+ legend=False, ax=ax, split=True, inner=None,
+ color=".8")
+
+ # --- Plot Data ---
+ # Find the x,y coordinates for each point
+ x_coords = np.arange(x_coords.shape[0])
+ first_header = list(self.measured_data)[0]
+ obs_data = self.measured_data.round({first_header: 6})
+ sns.pointplot(
+ x=first_header, y=out_name, color='g', markers='x',
+ linestyles='', capsize=16, data=obs_data, ax=ax)
+
+ ax.errorbar(
+ x_coords, obs_data[out_name].values,
+ yerr=1.96*self.measurement_error[out_name],
+ ecolor='g', fmt=' ', zorder=-1)
+
+ # Add labels to the legend
+ handles, labels = ax.get_legend_handles_labels()
+ labels.append('Data')
+
+ data_marker = mlines.Line2D(
+ [], [], color='lime', marker='+', linestyle='None',
+ markersize=10)
+ handles.append(data_marker)
+
+ # Add legend
+ ax.legend(handles=handles, labels=labels, loc='best',
+ fontsize='large', frameon=True)
+
+ else:
+ # Load posterior predictive
+ f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+
+ try:
+ x_coords = np.array(f[f"x_values/{out_name}"])
+ except:
+ x_coords = np.array(f["x_values"])
+
+ mu = np.mean(np.array(f[f"EDY/{out_name}"]), axis=0)
+ std = np.std(np.array(f[f"EDY/{out_name}"]), axis=0)
+
+ # --- Plot posterior predictive ---
+ plt.plot(
+ x_coords, mu, marker='o', color='b',
+ label='Mean Post. Predictive')
+ plt.fill_between(
+ x_coords, mu-1.96*std, mu+1.96*std, color='b',
+ alpha=0.15)
+
+ # --- Plot Data ---
+ ax.plot(
+ x_coords, self.measured_data[out_name].values,
+ 'ko', label='data', markeredgecolor='w')
+
+ # --- Plot ExpDesign ---
+ orig_ED_Y = self.engine.ExpDesign.Y[out_name]
+ for output in orig_ED_Y:
+ plt.plot(
+ x_coords, output, color='grey', alpha=0.15
+ )
+
+ # Add labels for axes
+ plt.xlabel('Time [s]')
+ plt.ylabel(out_name)
+
+ # Add labels to the legend
+ handles, labels = ax.get_legend_handles_labels()
+
+ patch = Patch(color='b', alpha=0.15)
+ handles.insert(1, patch)
+ labels.insert(1, '95 $\\%$ CI')
+
+ # Add legend
+ ax.legend(handles=handles, labels=labels, loc='best',
+ frameon=True)
+
+ # Save figure in pdf format
+ if self.emulator:
+ plotname = f'/Post_Prior_Perd_{Model.name}_emulator'
+ else:
+ plotname = f'/Post_Prior_Perd_{Model.name}'
+
+ fig.savefig(f'./{out_dir}{plotname}_{out_name}.pdf',
+ bbox_inches='tight')
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_model_comparison.py b/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_model_comparison.py
new file mode 100644
index 0000000000000000000000000000000000000000..769ad2ceaaced2c1fb6f18d22a9ca27278c3e8a1
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayes_inference/bayes_model_comparison.py
@@ -0,0 +1,680 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import emcee
+import numpy as np
+import os
+from scipy import stats
+import seaborn as sns
+import matplotlib.patches as patches
+import matplotlib.colors as mcolors
+import matplotlib.pylab as plt
+from .bayes_inference import BayesInference
+
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class BayesModelComparison:
+ """
+ A class to perform Bayesian Analysis.
+
+
+ Attributes
+ ----------
+ justifiability : bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `True`.
+ perturbed_data : array of shape (n_bootstrap_itrs, n_obs), optional
+ User defined perturbed data. The default is `None`.
+ n_bootstrap : int
+ Number of bootstrap iteration. The default is `1000`.
+ data_noise_level : float
+ A noise level to perturb the data set. The default is `0.01`.
+ just_n_meas : int
+ Number of measurements considered for visualization of the
+ justifiability results.
+
+ """
+
+ def __init__(self, justifiability=True, perturbed_data=None,
+ n_bootstrap=1000, data_noise_level=0.01, just_n_meas=2):
+
+ # TODO: check valid ranges of the parameters
+
+ self.justifiability = justifiability
+ self.perturbed_data = perturbed_data
+ self.n_bootstrap = n_bootstrap
+ self.data_noise_level = data_noise_level
+ self.just_n_meas = just_n_meas # TODO: what is this parameter?
+
+ # --------------------------------------------------------------------------
+ def create_model_comparison(self, model_dict, opts_dict):
+ """
+ Starts the two-stage model comparison.
+ Stage I: Compare models using Bayes factors.
+ Stage II: Compare models via justifiability analysis.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ opts_dict : dict
+ A dictionary given the `BayesInference` options.
+
+ Example:
+
+ >>> opts_bootstrap = {
+ "bootstrap": True,
+ "n_samples": 10000,
+ "Discrepancy": DiscrepancyOpts,
+ "emulator": True,
+ "plot_post_pred": True
+ }
+
+ Returns
+ -------
+ output : dict
+ A dictionary containing the objects and the model weights for the
+ comparison using Bayes factors and justifiability analysis.
+
+ """
+ # TODO: why are these two separate calls of the same function?
+ # They should be performable at the same time
+
+ # Bayes factor
+ bayes_dict_bf, model_weights_dict_bf = self.compare_models(
+ model_dict, opts_dict
+ )
+
+ output = {
+ 'Bayes objects BF': bayes_dict_bf,
+ 'Model weights BF': model_weights_dict_bf
+ }
+
+ # Justifiability analysis
+ if self.justifiability:
+ bayes_dict_ja, model_weights_dict_ja = self.compare_models(
+ model_dict, opts_dict, justifiability=True
+ )
+
+ # TODO: why does this version of the call not return a summarized confusion matrix?
+ output['Bayes objects JA'] = bayes_dict_ja
+ output['Model weights JA'] = model_weights_dict_ja
+
+ return output
+
+ # --------------------------------------------------------------------------
+ def compare_models(self, model_dict, opts_dict, justifiability=False):
+ """
+ Passes the options to instantiates the BayesInference class for each
+ model and passes the options from `opts_dict`. Then, it starts the
+ computations.
+ It also creates a folder and saves the diagrams, e.g., Bayes factor
+ plot, confusion matrix, etc.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ opts_dict : dict
+ A dictionary given the `BayesInference` options.
+ justifiability : bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `False`.
+
+ Returns
+ -------
+ bayes_dict : dict
+ A dictionary with `BayesInference` objects.
+ model_weights_dict : dict
+ A dictionary containing the model weights.
+
+ """
+
+ if not isinstance(model_dict, dict):
+ raise Exception("To run model comparsion, you need to pass a "
+ "dictionary of models.")
+
+ # Extract model names
+ self.model_names = [*model_dict]
+
+ # Compute total number of the measurement points
+ # TODO: there could be a different option for this here
+ Engine = list(model_dict.items())[0][1]
+ Engine.Model.read_observation()
+ self.n_meas = Engine.Model.n_obs
+
+ # ----- Generate data -----
+ # Find n_bootstrap
+ if self.perturbed_data is None:
+ n_bootstrap = self.n_bootstrap
+ else:
+ n_bootstrap = self.perturbed_data.shape[0]
+
+ # Create dataset
+ justData = self.generate_dataset(
+ model_dict, justifiability, n_bootstarp=n_bootstrap)
+
+ # Run create Interface for each model
+ self.bayes_dict = {}
+ for model in model_dict.keys():
+ print("-"*20)
+ print("Bayesian inference of {}.\n".format(model))
+
+ BayesOpts = BayesInference(model_dict[model])
+
+ # Explicitly set the settings of the BayesOpts
+ if self.use_Bayes_settings:
+ BayesOpts.emulator= True
+ BayesOpts.plot_post_pred = True
+ #BayesOpts.inference_method = 'rejection'
+ BayesOpts.bootstrap = True
+ BayesOpts.n_bootstrap_itrs = 10
+ BayesOpts.bootstrap_noise = 0.05
+
+ # Set the MCMC parameters
+ BayesOpts.inference_method = "MCMC"
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e3,#5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'multiprocessing': False,
+ 'verbose': False
+ }
+
+ # Set BayesInference options
+ for key, value in opts_dict.items():
+ if key in BayesOpts.__dict__.keys():
+ if key == "Discrepancy" and isinstance(value, dict):
+ setattr(BayesOpts, key, value[model])
+ else:
+ setattr(BayesOpts, key, value)
+
+ # Pass justifiability data as perturbed data
+ BayesOpts.perturbed_data = justData
+ BayesOpts.just_analysis = justifiability
+
+ self.bayes_dict[model] = BayesOpts.create_inference()
+ print("-"*20)
+
+ # Compute model weights
+ self.BME_Dict = dict()
+ for modelName, bayesObj in self.bayes_dict.items():
+ self.BME_Dict[modelName] = np.exp(bayesObj.log_BME, dtype=np.longdouble)#float128)
+
+ # BME correction in BayesInference class
+ self.model_weights = self.cal_model_weight(
+ self.BME_Dict, justifiability, n_bootstarp=n_bootstrap)
+
+ # Plot model weights
+ if justifiability:
+ model_names = self.model_names
+ model_names.insert(0, 'Observation')
+
+ # Split the model weights and save in a dict
+ list_ModelWeights = np.split(
+ self.model_weights, self.model_weights.shape[1]/self.n_meas, axis=1)
+ model_weights_dict = {key: weights for key, weights in
+ zip(model_names, list_ModelWeights)}
+
+ #self.plot_just_analysis(model_weights_dict)
+ else:
+ # Create box plot for model weights
+ self.plot_model_weights(self.model_weights, 'model_weights')
+
+ # Create kde plot for bayes factors
+ self.plot_bayes_factor(self.BME_Dict, 'kde_plot')
+
+ # Store model weights in a dict
+ model_weights_dict = {key: weights for key, weights in
+ zip(self.model_names, self.model_weights)}
+
+ return self.bayes_dict, model_weights_dict
+
+ # -------------------------------------------------------------------------
+ def generate_dataset(self, model_dict, justifiability=False,
+ n_bootstrap=1):
+ """
+ Generates the perturbed data set for the Bayes factor calculations and
+ the data set for the justifiability analysis.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `False`.
+ n_bootstrap : int, optional
+ Number of bootstrap iterations. The default is `1`.
+
+ Returns
+ -------
+ all_just_data: array
+ Created data set.
+
+ """
+ # Compute some variables
+ all_just_data = []
+ Engine = list(model_dict.items())[0][1]
+ out_names = Engine.Model.Output.names
+
+ # Perturb observations for Bayes Factor
+ if self.perturbed_data is None:
+ self.perturbed_data = self.__perturb_data(
+ Engine.Model.observations, out_names, n_bootstrap,
+ noise_level=self.data_noise_level)
+
+ # Only for Bayes Factor
+ if not justifiability:
+ return self.perturbed_data
+
+ # Evaluate metamodel
+ runs = {}
+ for key, metaModel in model_dict.items():
+ y_hat, _ = metaModel.eval_metamodel(nsamples=n_bootstrap)
+ runs[key] = y_hat
+
+ # Generate data
+ for i in range(n_bootstrap):
+ y_data = self.perturbed_data[i].reshape(1, -1)
+ justData = np.tril(np.repeat(y_data, y_data.shape[1], axis=0))
+ # Use surrogate runs for data-generating process
+ for key, metaModel in model_dict.items():
+ model_data = np.array(
+ [runs[key][out][i] for out in out_names]).reshape(y_data.shape)
+ justData = np.vstack((
+ justData,
+ np.tril(np.repeat(model_data, model_data.shape[1], axis=0))
+ ))
+ # Save in a list
+ all_just_data.append(justData)
+
+ # Squeeze the array
+ all_just_data = np.array(all_just_data).transpose(1, 0, 2).reshape(
+ -1, np.array(all_just_data).shape[2]
+ )
+
+ return all_just_data
+
+ # -------------------------------------------------------------------------
+ def __perturb_data(self, data, output_names, n_bootstrap, noise_level):
+ """
+ Returns an array with n_bootstrap_itrs rowsof perturbed data.
+ The first row includes the original observation data.
+ If `self.bayes_loocv` is True, a 2d-array will be returned with
+ repeated rows and zero diagonal entries.
+
+ Parameters
+ ----------
+ data : pandas DataFrame
+ Observation data.
+ output_names : list
+ List of the output names.
+
+ Returns
+ -------
+ final_data : array
+ Perturbed data set.
+
+ """
+ obs_data = data[output_names].values
+ n_measurement, n_outs = obs_data.shape
+ n_tot_measurement = obs_data[~np.isnan(obs_data)].shape[0]
+ final_data = np.zeros(
+ (n_bootstrap, n_tot_measurement)
+ )
+ final_data[0] = obs_data.T[~np.isnan(obs_data.T)]
+ for itrIdx in range(1, n_bootstrap):
+ data = np.zeros((n_measurement, n_outs))
+ for idx in range(len(output_names)):
+ std = np.nanstd(obs_data[:, idx])
+ if std == 0:
+ std = 0.001
+ noise = std * noise_level
+ data[:, idx] = np.add(
+ obs_data[:, idx],
+ np.random.normal(0, 1, obs_data.shape[0]) * noise,
+ )
+
+ final_data[itrIdx] = data.T[~np.isnan(data.T)]
+
+ return final_data
+
+ # -------------------------------------------------------------------------
+ def cal_model_weight(self, BME_Dict, justifiability=False, n_bootstrap=1):
+ """
+ Normalize the BME (Asumption: Model Prior weights are equal for models)
+
+ Parameters
+ ----------
+ BME_Dict : dict
+ A dictionary containing the BME values.
+
+ Returns
+ -------
+ model_weights : array
+ Model weights.
+
+ """
+ # Stack the BME values for all models
+ all_BME = np.vstack(list(BME_Dict.values()))
+
+ if justifiability:
+ # Compute expected log_BME for justifiabiliy analysis
+ all_BME = all_BME.reshape(
+ all_BME.shape[0], -1, n_bootstrap).mean(axis=2)
+
+ # Model weights
+ model_weights = np.divide(all_BME, np.nansum(all_BME, axis=0))
+
+ return model_weights
+
+ # -------------------------------------------------------------------------
+ def plot_just_analysis(self, model_weights_dict):
+ """
+ Visualizes the confusion matrix and the model wights for the
+ justifiability analysis.
+
+ Parameters
+ ----------
+ model_weights_dict : dict
+ Model weights.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+ Color = [*mcolors.TABLEAU_COLORS]
+ names = [*model_weights_dict]
+
+ model_names = [model.replace('_', '$-$') for model in self.model_names]
+ for name in names:
+ fig, ax = plt.subplots()
+ for i, model in enumerate(model_names[1:]):
+ plt.plot(list(range(1, self.n_meas+1)),
+ model_weights_dict[name][i],
+ color=Color[i], marker='o',
+ ms=10, linewidth=2, label=model
+ )
+
+ plt.title(f"Data generated by: {name.replace('_', '$-$')}")
+ plt.ylabel("Weights")
+ plt.xlabel("No. of measurement points")
+ ax.set_xticks(list(range(1, self.n_meas+1)))
+ plt.legend(loc="best")
+ fig.savefig(
+ f'{directory}modelWeights_{name}.svg', bbox_inches='tight'
+ )
+ plt.close()
+
+ # Confusion matrix for some measurement points
+ epsilon = 1 if self.just_n_meas != 1 else 0
+ for index in range(0, self.n_meas+epsilon, self.just_n_meas):
+ weights = np.array(
+ [model_weights_dict[key][:, index] for key in model_weights_dict]
+ )
+ g = sns.heatmap(
+ weights.T, annot=True, cmap='Blues', xticklabels=model_names,
+ yticklabels=model_names[1:], annot_kws={"size": 24}
+ )
+
+ # x axis on top
+ g.xaxis.tick_top()
+ g.xaxis.set_label_position('top')
+ g.set_xlabel(r"\textbf{Data generated by:}", labelpad=15)
+ g.set_ylabel(r"\textbf{Model weight for:}", labelpad=15)
+ g.figure.savefig(
+ f"{directory}confusionMatrix_ND_{index+1}.pdf",
+ bbox_inches='tight'
+ )
+ plt.close()
+
+ # -------------------------------------------------------------------------
+ def plot_model_weights(self, model_weights, plot_name):
+ """
+ Visualizes the model weights resulting from BMS via the observation
+ data.
+
+ Parameters
+ ----------
+ model_weights : array
+ Model weights.
+ plot_name : str
+ Plot name.
+
+ Returns
+ -------
+ None.
+
+ """
+ font_size = 40
+ # mkdir for plots
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+
+ # Create figure
+ fig, ax = plt.subplots()
+
+ # Filter data using np.isnan
+ mask = ~np.isnan(model_weights.T)
+ filtered_data = [d[m] for d, m in zip(model_weights, mask.T)]
+
+ # Create the boxplot
+ bp = ax.boxplot(filtered_data, patch_artist=True, showfliers=False)
+
+ # change outline color, fill color and linewidth of the boxes
+ for box in bp['boxes']:
+ # change outline color
+ box.set(color='#7570b3', linewidth=4)
+ # change fill color
+ box.set(facecolor='#1b9e77')
+
+ # change color and linewidth of the whiskers
+ for whisker in bp['whiskers']:
+ whisker.set(color='#7570b3', linewidth=2)
+
+ # change color and linewidth of the caps
+ for cap in bp['caps']:
+ cap.set(color='#7570b3', linewidth=2)
+
+ # change color and linewidth of the medians
+ for median in bp['medians']:
+ median.set(color='#b2df8a', linewidth=2)
+
+ # change the style of fliers and their fill
+ # for flier in bp['fliers']:
+ # flier.set(marker='o', color='#e7298a', alpha=0.75)
+
+ # Custom x-axis labels
+ model_names = [model.replace('_', '$-$') for model in self.model_names]
+ ax.set_xticklabels(model_names)
+
+ ax.set_ylabel('Weight', fontsize=font_size)
+
+ # Title
+ plt.title('Posterior Model Weights')
+
+ # Set y lim
+ ax.set_ylim((-0.05, 1.05))
+
+ # Set size of the ticks
+ for t in ax.get_xticklabels():
+ t.set_fontsize(font_size)
+ for t in ax.get_yticklabels():
+ t.set_fontsize(font_size)
+
+ # Save the figure
+ fig.savefig(
+ f'./{directory}{plot_name}.pdf', bbox_inches='tight'
+ )
+
+ plt.close()
+
+ # -------------------------------------------------------------------------
+ def plot_bayes_factor(self, BME_Dict, plot_name=''):
+ """
+ Plots the Bayes factor distibutions in a :math:`N_m \\times N_m`
+ matrix, where :math:`N_m` is the number of the models.
+
+ Parameters
+ ----------
+ BME_Dict : dict
+ A dictionary containing the BME values of the models.
+ plot_name : str, optional
+ Plot name. The default is ''.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ font_size = 40
+
+ # mkdir for plots
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+
+ Colors = ["blue", "green", "gray", "brown"]
+
+ model_names = list(BME_Dict.keys())
+ nModels = len(model_names)
+
+ # Plots
+ fig, axes = plt.subplots(
+ nrows=nModels, ncols=nModels, sharex=True, sharey=True
+ )
+
+ for i, key_i in enumerate(model_names):
+
+ for j, key_j in enumerate(model_names):
+ ax = axes[i, j]
+ # Set size of the ticks
+ for t in ax.get_xticklabels():
+ t.set_fontsize(font_size)
+ for t in ax.get_yticklabels():
+ t.set_fontsize(font_size)
+
+ if j != i:
+
+ # Null hypothesis: key_j is the better model
+ BayesFactor = np.log10(
+ np.divide(BME_Dict[key_i], BME_Dict[key_j])
+ )
+
+ # sns.kdeplot(BayesFactor, ax=ax, color=Colors[i], shade=True)
+ # sns.histplot(BayesFactor, ax=ax, stat="probability",
+ # kde=True, element='step',
+ # color=Colors[j])
+
+ # taken from seaborn's source code (utils.py and
+ # distributions.py)
+ def seaborn_kde_support(data, bw, gridsize, cut, clip):
+ if clip is None:
+ clip = (-np.inf, np.inf)
+ support_min = max(data.min() - bw * cut, clip[0])
+ support_max = min(data.max() + bw * cut, clip[1])
+ return np.linspace(support_min, support_max, gridsize)
+
+ kde_estim = stats.gaussian_kde(
+ BayesFactor, bw_method='scott'
+ )
+
+ # manual linearization of data
+ # linearized = np.linspace(
+ # quotient.min(), quotient.max(), num=500)
+
+ # or better: mimic seaborn's internal stuff
+ bw = kde_estim.scotts_factor() * np.std(BayesFactor)
+ linearized = seaborn_kde_support(
+ BayesFactor, bw, 100, 3, None)
+
+ # computes values of the estimated function on the
+ # estimated linearized inputs
+ Z = kde_estim.evaluate(linearized)
+
+ # https://stackoverflow.com/questions/29661574/normalize-
+ # numpy-array-columns-in-python
+ def normalize(x):
+ return (x - x.min(0)) / x.ptp(0)
+
+ # normalize so it is between 0;1
+ Z2 = normalize(Z)
+ ax.plot(linearized, Z2, "-", color=Colors[i], linewidth=4)
+ ax.fill_between(
+ linearized, 0, Z2, color=Colors[i], alpha=0.25
+ )
+
+ # Draw BF significant levels according to Jeffreys 1961
+ # Strong evidence for both models
+ ax.axvline(
+ x=np.log10(3), ymin=0, linewidth=4, color='dimgrey'
+ )
+ # Strong evidence for one model
+ ax.axvline(
+ x=np.log10(10), ymin=0, linewidth=4, color='orange'
+ )
+ # Decisive evidence for one model
+ ax.axvline(
+ x=np.log10(100), ymin=0, linewidth=4, color='r'
+ )
+
+ # legend
+ BF_label = key_i.replace('_', '$-$') + \
+ '/' + key_j.replace('_', '$-$')
+ legend_elements = [
+ patches.Patch(facecolor=Colors[i], edgecolor=Colors[i],
+ label=f'BF({BF_label})')
+ ]
+ ax.legend(
+ loc='upper left', handles=legend_elements,
+ fontsize=font_size-(nModels+1)*5
+ )
+
+ elif j == i:
+ # build a rectangle in axes coords
+ left, width = 0, 1
+ bottom, height = 0, 1
+
+ # axes coordinates are 0,0 is bottom left and 1,1 is upper
+ # right
+ p = patches.Rectangle(
+ (left, bottom), width, height, color='white',
+ fill=True, transform=ax.transAxes, clip_on=False
+ )
+ ax.grid(False)
+ ax.add_patch(p)
+ # ax.text(0.5*(left+right), 0.5*(bottom+top), key_i,
+ fsize = font_size+20 if nModels < 4 else font_size
+ ax.text(0.5, 0.5, key_i.replace('_', '$-$'),
+ horizontalalignment='center',
+ verticalalignment='center',
+ fontsize=fsize, color=Colors[i],
+ transform=ax.transAxes)
+
+ # Defining custom 'ylim' values.
+ custom_ylim = (0, 1.05)
+
+ # Setting the values for all axes.
+ plt.setp(axes, ylim=custom_ylim)
+
+ # set labels
+ for i in range(nModels):
+ axes[-1, i].set_xlabel('log$_{10}$(BF)', fontsize=font_size)
+ axes[i, 0].set_ylabel('Probability', fontsize=font_size)
+
+ # Adjust subplots
+ plt.subplots_adjust(wspace=0.2, hspace=0.1)
+
+ plt.savefig(
+ f'./{directory}Bayes_Factor{plot_name}.pdf', bbox_inches='tight'
+ )
+
+ plt.close()
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/discrepancy.py b/examples/analytical-function/bayesvalidrox/bayes_inference/discrepancy.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3c235ebeb6d6ae9e109ca862cc522cc21efb45e
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayes_inference/discrepancy.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import scipy.stats as stats
+from bayesvalidrox.surrogate_models.exp_designs import ExpDesigns
+
+
+class Discrepancy:
+ """
+ Discrepancy class for Bayesian inference method.
+ We define the reference or reality to be equal to what we can model and a
+ descripancy term \\( \\epsilon \\). We consider the followin format:
+
+ $$\\textbf{y}_{\\text{reality}} = \\mathcal{M}(\\theta) + \\epsilon,$$
+
+ where \\( \\epsilon \\in R^{N_{out}} \\) represents the the effects of
+ measurement error and model inaccuracy. For simplicity, it can be defined
+ as an additive Gaussian disrepancy with zeromean and given covariance
+ matrix \\( \\Sigma \\):
+
+ $$\\epsilon \\sim \\mathcal{N}(\\epsilon|0, \\Sigma). $$
+
+ In the context of model inversion or calibration, an observation point
+ \\( \\textbf{y}_i \\in \\mathcal{y} \\) is a realization of a Gaussian
+ distribution with mean value of \\(\\mathcal{M}(\\theta) \\) and covariance
+ matrix of \\( \\Sigma \\).
+
+ $$ p(\\textbf{y}|\\theta) = \\mathcal{N}(\\textbf{y}|\\mathcal{M}
+ (\\theta))$$
+
+ The following options are available:
+
+ * Option A: With known redidual covariance matrix \\(\\Sigma\\) for
+ independent measurements.
+
+ * Option B: With unknown redidual covariance matrix \\(\\Sigma\\),
+ paramethrized as \\(\\Sigma(\\theta_{\\epsilon})=\\sigma^2 \\textbf{I}_
+ {N_{out}}\\) with unknown residual variances \\(\\sigma^2\\).
+ This term will be jointly infered with the uncertain input parameters. For
+ the inversion, you need to define a prior marginal via `Input` class. Note
+ that \\(\\sigma^2\\) is only a single scalar multiplier for the diagonal
+ entries of the covariance matrix \\(\\Sigma\\).
+
+ Attributes
+ ----------
+ InputDisc : obj
+ Input object. When the \\(\\sigma^2\\) is expected to be inferred
+ jointly with the parameters (`Option B`).If multiple output groups are
+ defined by `Model.Output.names`, each model output needs to have.
+ a prior marginal using the `Input` class. The default is `''`.
+ disc_type : str
+ Type of the noise definition. `'Gaussian'` is only supported so far.
+ parameters : dict or pandas.DataFrame
+ Known residual variance \\(\\sigma^2\\), i.e. diagonal entry of the
+ covariance matrix of the multivariate normal likelihood in case of
+ `Option A`.
+
+ """
+
+ def __init__(self, InputDisc='', disc_type='Gaussian', parameters=None):
+ # Set the values
+ self.InputDisc = InputDisc
+ self.disc_type = disc_type
+ self.parameters = parameters
+
+ # Other inits
+ self.ExpDesign = None
+ self.n_samples = None
+ self.sigma2_prior = None
+ self.name = None
+ self.opt_sigma = None # This will be set in the inference class and used in mcmc
+ # -------------------------------------------------------------------------
+ def get_sample(self, n_samples):
+ """
+ Generate samples for the \\(\\sigma^2\\), i.e. the diagonal entries of
+ the variance-covariance matrix in the multivariate normal distribution.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of samples (parameter sets).
+
+ Returns
+ -------
+ sigma2_prior: array of shape (n_samples, n_params)
+ \\(\\sigma^2\\) samples.
+
+ """
+ self.n_samples = n_samples # TODO: not used again in here - needed from the outside?
+
+ if self.InputDisc == '':
+ raise AttributeError('Cannot create new samples, please provide input distributions')
+
+ # Create and store BoundTuples
+ self.ExpDesign = ExpDesigns(self.InputDisc)
+ self.ExpDesign.sampling_method = 'random'
+
+ # TODO: why does it call 'generate_ED' instead of 'generate_samples?
+ # ExpDesign.bound_tuples, onp_sigma, prior_space needed from the outside
+ # Discrepancy opt_sigma, InputDisc needed from the outside
+ # TODO: opt_sigma not defined here, but called from the outside??
+ self.ExpDesign.generate_ED(
+ n_samples, max_pce_deg=1
+ )
+ # TODO: need to recheck the following line
+ # This used to simply be the return from the call above
+ self.sigma2_prior = self.ExpDesign.X
+
+ # Naive approach: Fit a gaussian kernel to the provided data
+ self.ExpDesign.JDist = stats.gaussian_kde(self.ExpDesign.raw_data)
+
+ # Save the names of sigmas
+ if len(self.InputDisc.Marginals) != 0:
+ self.name = []
+ for Marginalidx in range(len(self.InputDisc.Marginals)):
+ self.name.append(self.InputDisc.Marginals[Marginalidx].name)
+
+ return self.sigma2_prior
diff --git a/examples/analytical-function/bayesvalidrox/bayes_inference/mcmc.py b/examples/analytical-function/bayesvalidrox/bayes_inference/mcmc.py
new file mode 100644
index 0000000000000000000000000000000000000000..d78d15b5fd90dc4477da7d0fd58da835acc75310
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayes_inference/mcmc.py
@@ -0,0 +1,909 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import numpy as np
+import emcee
+import pandas as pd
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_pdf import PdfPages
+import multiprocessing
+import scipy.stats as st
+from scipy.linalg import cholesky as chol
+import warnings
+import shutil
+os.environ["OMP_NUM_THREADS"] = "1"
+
+
+class MCMC:
+ """
+ A class for bayesian inference via a Markov-Chain Monte-Carlo (MCMC)
+ Sampler to approximate the posterior distribution of the Bayes theorem:
+ $$p(\\theta|\\mathcal{y}) = \\frac{p(\\mathcal{y}|\\theta) p(\\theta)}
+ {p(\\mathcal{y})}.$$
+
+ This class make inference with emcee package [1] using an Affine Invariant
+ Ensemble sampler (AIES) [2].
+
+ [1] Foreman-Mackey, D., Hogg, D.W., Lang, D. and Goodman, J., 2013.emcee:
+ the MCMC hammer. Publications of the Astronomical Society of the
+ Pacific, 125(925), p.306. https://emcee.readthedocs.io/en/stable/
+
+ [2] Goodman, J. and Weare, J., 2010. Ensemble samplers with affine
+ invariance. Communications in applied mathematics and computational
+ science, 5(1), pp.65-80.
+
+
+ Attributes
+ ----------
+ BayesOpts : obj
+ Bayes object.
+ """
+
+ def __init__(self, BayesOpts):
+
+ self.BayesOpts = BayesOpts
+
+ def run_sampler(self, observation, total_sigma2):
+
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.engine.MetaModel
+ Model = BayesObj.engine.Model
+ Discrepancy = self.BayesOpts.Discrepancy
+ n_cpus = Model.n_cpus
+ priorDist = BayesObj.engine.ExpDesign.JDist
+ ndim = MetaModel.n_params
+ self.counter = 0
+ output_dir = f'Outputs_Bayes_{Model.name}_{self.BayesOpts.name}'
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ self.observation = observation
+ self.total_sigma2 = total_sigma2
+
+ # Unpack mcmc parameters given to BayesObj.mcmc_params
+ self.initsamples = None
+ self.nwalkers = 100
+ self.nburn = 200
+ self.nsteps = 100000
+ self.moves = None
+ self.mp = False
+ self.verbose = False
+
+ # Extract initial samples
+ if 'init_samples' in BayesObj.mcmc_params:
+ self.initsamples = BayesObj.mcmc_params['init_samples']
+ if isinstance(self.initsamples, pd.DataFrame):
+ self.initsamples = self.initsamples.values
+
+ # Extract number of steps per walker
+ if 'n_steps' in BayesObj.mcmc_params:
+ self.nsteps = int(BayesObj.mcmc_params['n_steps'])
+ # Extract number of walkers (chains)
+ if 'n_walkers' in BayesObj.mcmc_params:
+ self.nwalkers = int(BayesObj.mcmc_params['n_walkers'])
+ # Extract moves
+ if 'moves' in BayesObj.mcmc_params:
+ self.moves = BayesObj.mcmc_params['moves']
+ # Extract multiprocessing
+ if 'multiprocessing' in BayesObj.mcmc_params:
+ self.mp = BayesObj.mcmc_params['multiprocessing']
+ # Extract verbose
+ if 'verbose' in BayesObj.mcmc_params:
+ self.verbose = BayesObj.mcmc_params['verbose']
+
+ # Set initial samples
+ np.random.seed(0)
+ if self.initsamples is None:
+ try:
+ initsamples = priorDist.sample(self.nwalkers).T
+ except:
+ # when aPCE selected - gaussian kernel distribution
+ inputSamples = self.BayesOpts.engine.ExpDesign.raw_data.T
+ random_indices = np.random.choice(
+ len(inputSamples), size=self.nwalkers, replace=False
+ )
+ initsamples = inputSamples[random_indices]
+
+ else:
+ if self.initsamples.ndim == 1:
+ # When MAL is given.
+ theta = self.initsamples
+ initsamples = [theta + 1e-1*np.multiply(
+ np.random.randn(ndim), theta) for i in
+ range(self.nwalkers)]
+ else:
+ # Pick samples based on a uniform dist between min and max of
+ # each dim
+ initsamples = np.zeros((self.nwalkers, ndim))
+ bound_tuples = []
+ for idx_dim in range(ndim):
+ lower = np.min(self.initsamples[:, idx_dim])
+ upper = np.max(self.initsamples[:, idx_dim])
+ bound_tuples.append((lower, upper))
+ dist = st.uniform(loc=lower, scale=upper-lower)
+ initsamples[:, idx_dim] = dist.rvs(size=self.nwalkers)
+
+ # Update lower and upper
+ MetaModel.ExpDesign.bound_tuples = bound_tuples
+
+ # Check if sigma^2 needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ sigma2_samples = Discrepancy.get_sample(self.nwalkers)
+
+ # Update initsamples
+ initsamples = np.hstack((initsamples, sigma2_samples))
+
+ # Update ndim
+ ndim = initsamples.shape[1]
+
+ # Discrepancy bound
+ disc_bound_tuple = Discrepancy.ExpDesign.bound_tuples
+
+ # Update bound_tuples
+ BayesObj.engine.ExpDesign.bound_tuples += disc_bound_tuple
+
+ print("\n>>>> Bayesian inference with MCMC for "
+ f"{self.BayesOpts.name} started. <<<<<<")
+
+ # Set up the backend
+ filename = f"{output_dir}/emcee_sampler.h5"
+ backend = emcee.backends.HDFBackend(filename)
+ # Clear the backend in case the file already exists
+ backend.reset(self.nwalkers, ndim)
+
+ # Define emcee sampler
+ # Here we'll set up the computation. emcee combines multiple "walkers",
+ # each of which is its own MCMC chain. The number of trace results will
+ # be nwalkers * nsteps.
+ if self.mp:
+ # Run in parallel
+ if n_cpus is None:
+ n_cpus = multiprocessing.cpu_count()
+
+ with multiprocessing.Pool(n_cpus) as pool:
+ sampler = emcee.EnsembleSampler(
+ self.nwalkers, ndim, self.log_posterior, moves=self.moves,
+ pool=pool, backend=backend
+ )
+
+ # Check if a burn-in phase is needed!
+ if self.initsamples is None:
+ # Burn-in
+ print("\n Burn-in period is starting:")
+ pos = sampler.run_mcmc(
+ initsamples, self.nburn, progress=True
+ )
+
+ # Reset sampler
+ sampler.reset()
+ pos = pos.coords
+ else:
+ pos = initsamples
+
+ # Production run
+ print("\n Production run is starting:")
+ pos, prob, state = sampler.run_mcmc(
+ pos, self.nsteps, progress=True
+ )
+
+ else:
+ # Run in series and monitor the convergence
+ sampler = emcee.EnsembleSampler(
+ self.nwalkers, ndim, self.log_posterior, moves=self.moves,
+ backend=backend, vectorize=True
+ )
+
+ # Check if a burn-in phase is needed!
+ if self.initsamples is None:
+ # Burn-in
+ print("\n Burn-in period is starting:")
+ pos = sampler.run_mcmc(
+ initsamples, self.nburn, progress=True
+ )
+
+ # Reset sampler
+ sampler.reset()
+ pos = pos.coords
+ else:
+ pos = initsamples
+
+ # Production run
+ print("\n Production run is starting:")
+
+ # Track how the average autocorrelation time estimate changes
+ autocorrIdx = 0
+ autocorr = np.empty(self.nsteps)
+ tauold = np.inf
+ autocorreverynsteps = 50
+
+ # sample step by step using the generator sampler.sample
+ for sample in sampler.sample(pos,
+ iterations=self.nsteps,
+ tune=True,
+ progress=True):
+
+ # only check convergence every autocorreverynsteps steps
+ if sampler.iteration % autocorreverynsteps:
+ continue
+
+ # Train model discrepancy/error
+ if hasattr(BayesObj, 'errorModel') and BayesObj.errorModel \
+ and not sampler.iteration % 3 * autocorreverynsteps:
+ try:
+ self.error_MetaModel = self.train_error_model(sampler)
+ except:
+ pass
+
+ # Print the current mean acceptance fraction
+ if self.verbose:
+ print("\nStep: {}".format(sampler.iteration))
+ acc_fr = np.mean(sampler.acceptance_fraction)
+ print(f"Mean acceptance fraction: {acc_fr:.3f}")
+
+ # compute the autocorrelation time so far
+ # using tol=0 means that we'll always get an estimate even if
+ # it isn't trustworthy
+ tau = sampler.get_autocorr_time(tol=0)
+ # average over walkers
+ autocorr[autocorrIdx] = np.nanmean(tau)
+ autocorrIdx += 1
+
+ # output current autocorrelation estimate
+ if self.verbose:
+ print(f"Mean autocorr. time estimate: {np.nanmean(tau):.3f}")
+ list_gr = np.round(self.gelman_rubin(sampler.chain), 3)
+ print("Gelman-Rubin Test*: ", list_gr)
+
+ # check convergence
+ converged = np.all(tau*autocorreverynsteps < sampler.iteration)
+ converged &= np.all(np.abs(tauold - tau) / tau < 0.01)
+ converged &= np.all(self.gelman_rubin(sampler.chain) < 1.1)
+
+ if converged:
+ break
+ tauold = tau
+
+ # Posterior diagnostics
+ try:
+ tau = sampler.get_autocorr_time(tol=0)
+ except emcee.autocorr.AutocorrError:
+ tau = 5
+
+ if all(np.isnan(tau)):
+ tau = 5
+
+ burnin = int(2*np.nanmax(tau))
+ thin = int(0.5*np.nanmin(tau)) if int(0.5*np.nanmin(tau)) != 0 else 1
+ finalsamples = sampler.get_chain(discard=burnin, flat=True, thin=thin)
+ acc_fr = np.nanmean(sampler.acceptance_fraction)
+ list_gr = np.round(self.gelman_rubin(sampler.chain[:, burnin:]), 3)
+
+ # Print summary
+ print('\n')
+ print('-'*15 + 'Posterior diagnostics' + '-'*15)
+ print(f"Mean auto-correlation time: {np.nanmean(tau):.3f}")
+ print(f"Thin: {thin}")
+ print(f"Burn-in: {burnin}")
+ print(f"Flat chain shape: {finalsamples.shape}")
+ print(f"Mean acceptance fraction*: {acc_fr:.3f}")
+ print("Gelman-Rubin Test**: ", list_gr)
+
+ print("\n* This value must lay between 0.234 and 0.5.")
+ print("** These values must be smaller than 1.1.")
+ print('-'*50)
+
+ print(f"\n>>>> Bayesian inference with MCMC for {self.BayesOpts.name} "
+ "successfully completed. <<<<<<\n")
+
+ # Extract parameter names and their prior ranges
+ par_names = self.BayesOpts.engine.ExpDesign.par_names
+
+ if Discrepancy.opt_sigma != 'B':
+ for i in range(len(Discrepancy.InputDisc.Marginals)):
+ par_names.append(Discrepancy.InputDisc.Marginals[i].name)
+
+ params_range = self.BayesOpts.engine.ExpDesign.bound_tuples
+
+ # Plot traces
+ if self.verbose and self.nsteps < 10000:
+ pdf = PdfPages(output_dir+'/traceplots.pdf')
+ fig = plt.figure()
+ for parIdx in range(ndim):
+ # Set up the axes with gridspec
+ fig = plt.figure()
+ grid = plt.GridSpec(4, 4, hspace=0.2, wspace=0.2)
+ main_ax = fig.add_subplot(grid[:-1, :3])
+ y_hist = fig.add_subplot(grid[:-1, -1], xticklabels=[],
+ sharey=main_ax)
+
+ for i in range(self.nwalkers):
+ samples = sampler.chain[i, :, parIdx]
+ main_ax.plot(samples, '-')
+
+ # histogram on the attached axes
+ y_hist.hist(samples[burnin:], 40, histtype='stepfilled',
+ orientation='horizontal', color='gray')
+
+ main_ax.set_ylim(params_range[parIdx])
+ main_ax.set_title('traceplot for ' + par_names[parIdx])
+ main_ax.set_xlabel('step number')
+
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ pdf.close()
+
+ # plot development of autocorrelation estimate
+ if not self.mp:
+ fig1 = plt.figure()
+ steps = autocorreverynsteps*np.arange(1, autocorrIdx+1)
+ taus = autocorr[:autocorrIdx]
+ plt.plot(steps, steps / autocorreverynsteps, "--k")
+ plt.plot(steps, taus)
+ plt.xlim(0, steps.max())
+ plt.ylim(0, np.nanmax(taus)+0.1*(np.nanmax(taus)-np.nanmin(taus)))
+ plt.xlabel("number of steps")
+ plt.ylabel(r"mean $\hat{\tau}$")
+ fig1.savefig(f"{output_dir}/autocorrelation_time.pdf",
+ bbox_inches='tight')
+
+ # logml_dict = self.marginal_llk_emcee(sampler, self.nburn, logp=None,
+ # maxiter=5000)
+ # print('\nThe Bridge Sampling Estimation is "
+ # f"{logml_dict['logml']:.5f}.')
+
+ # # Posterior-based expectation of posterior probablity
+ # postExpPostLikelihoods = np.mean(sampler.get_log_prob(flat=True)
+ # [self.nburn*self.nwalkers:])
+
+ # # Posterior-based expectation of prior densities
+ # postExpPrior = np.mean(self.log_prior(emcee_trace.T))
+
+ # # Posterior-based expectation of likelihoods
+ # postExpLikelihoods_emcee = postExpPostLikelihoods - postExpPrior
+
+ # # Calculate Kullback-Leibler Divergence
+ # KLD_emcee = postExpLikelihoods_emcee - logml_dict['logml']
+ # print("Kullback-Leibler divergence: %.5f"%KLD_emcee)
+
+ # # Information Entropy based on Entropy paper Eq. 38
+ # infEntropy_emcee = logml_dict['logml'] - postExpPrior -
+ # postExpLikelihoods_emcee
+ # print("Information Entropy: %.5f" %infEntropy_emcee)
+
+ Posterior_df = pd.DataFrame(finalsamples, columns=par_names)
+
+ return Posterior_df
+
+ # -------------------------------------------------------------------------
+ def log_prior(self, theta):
+ """
+ Calculates the log prior likelihood \\( p(\\theta)\\) for the given
+ parameter set(s) \\( \\theta \\).
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter sets, i.e. proposals of MCMC chains.
+
+ Returns
+ -------
+ logprior: float or array of shape n_samples
+ Log prior likelihood. If theta has only one row, a single value is
+ returned otherwise an array.
+
+ """
+
+ MetaModel = self.BayesOpts.MetaModel
+ Discrepancy = self.BayesOpts.Discrepancy
+
+ # Find the number of sigma2 parameters
+ if Discrepancy.opt_sigma != 'B':
+ disc_bound_tuples = Discrepancy.ExpDesign.bound_tuples
+ disc_marginals = Discrepancy.ExpDesign.InputObj.Marginals
+ disc_prior_space = Discrepancy.ExpDesign.prior_space
+ n_sigma2 = len(disc_bound_tuples)
+ else:
+ n_sigma2 = -len(theta)
+ prior_dist = self.BayesOpts.engine.ExpDesign.prior_space
+ params_range = self.BayesOpts.engine.ExpDesign.bound_tuples
+ theta = theta if theta.ndim != 1 else theta.reshape((1, -1))
+ nsamples = theta.shape[0]
+ logprior = -np.inf*np.ones(nsamples)
+
+ for i in range(nsamples):
+ # Check if the sample is within the parameters' range
+ if self._check_ranges(theta[i], params_range):
+ # Check if all dists are uniform, if yes priors are equal.
+ if all(MetaModel.input_obj.Marginals[i].dist_type == 'uniform'
+ for i in range(MetaModel.n_params)):
+ logprior[i] = 0.0
+ else:
+ logprior[i] = np.log(
+ prior_dist.pdf(theta[i, :-n_sigma2].T)
+ )
+
+ # Check if bias term needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ if self._check_ranges(theta[i, -n_sigma2:],
+ disc_bound_tuples):
+ if all('unif' in disc_marginals[i].dist_type for i in
+ range(Discrepancy.ExpDesign.ndim)):
+ logprior[i] = 0.0
+ else:
+ logprior[i] += np.log(
+ disc_prior_space.pdf(theta[i, -n_sigma2:])
+ )
+
+ if nsamples == 1:
+ return logprior[0]
+ else:
+ return logprior
+
+ # -------------------------------------------------------------------------
+ def log_likelihood(self, theta):
+ """
+ Computes likelihood \\( p(\\mathcal{Y}|\\theta)\\) of the performance
+ of the (meta-)model in reproducing the observation data.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ log_like : array of shape (n_samples)
+ Log likelihood.
+
+ """
+
+ BayesOpts = self.BayesOpts
+ MetaModel = BayesOpts.MetaModel
+ Discrepancy = self.BayesOpts.Discrepancy
+
+ # Find the number of sigma2 parameters
+ if Discrepancy.opt_sigma != 'B':
+ disc_bound_tuples = Discrepancy.ExpDesign.bound_tuples
+ n_sigma2 = len(disc_bound_tuples)
+ else:
+ n_sigma2 = -len(theta)
+ # Check if bias term needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ sigma2 = theta[:, -n_sigma2:]
+ theta = theta[:, :-n_sigma2]
+ else:
+ sigma2 = None
+ theta = theta if theta.ndim != 1 else theta.reshape((1, -1))
+
+ # Evaluate Model/MetaModel at theta
+ mean_pred, BayesOpts._std_pce_prior_pred = self.eval_model(theta)
+
+ # Surrogate model's error using RMSE of test data
+ surrError = MetaModel.rmse if hasattr(MetaModel, 'rmse') else None
+
+ # Likelihood
+ log_like = BayesOpts.normpdf(
+ mean_pred, self.observation, self.total_sigma2, sigma2,
+ std=surrError
+ )
+ return log_like
+
+ # -------------------------------------------------------------------------
+ def log_posterior(self, theta):
+ """
+ Computes the posterior likelihood \\(p(\\theta| \\mathcal{Y})\\) for
+ the given parameterset.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ log_like : array of shape (n_samples)
+ Log posterior likelihood.
+
+ """
+
+ nsamples = 1 if theta.ndim == 1 else theta.shape[0]
+
+ if nsamples == 1:
+ if self.log_prior(theta) == -np.inf:
+ return -np.inf
+ else:
+ # Compute log prior
+ log_prior = self.log_prior(theta)
+ # Compute log Likelihood
+ log_likelihood = self.log_likelihood(theta)
+
+ return log_prior + log_likelihood
+ else:
+ # Compute log prior
+ log_prior = self.log_prior(theta)
+
+ # Initialize log_likelihood
+ log_likelihood = -np.inf*np.ones(nsamples)
+
+ # find the indices for -inf sets
+ non_inf_idx = np.where(log_prior != -np.inf)[0]
+
+ # Compute loLikelihoods
+ if non_inf_idx.size != 0:
+ log_likelihood[non_inf_idx] = self.log_likelihood(
+ theta[non_inf_idx]
+ )
+
+ return log_prior + log_likelihood
+
+ # -------------------------------------------------------------------------
+ def eval_model(self, theta):
+ """
+ Evaluates the (meta-) model at the given theta.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean model prediction.
+ std_pred : dict
+ Std of model prediction.
+
+ """
+
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.MetaModel
+ Model = BayesObj.engine.Model
+
+ if BayesObj.emulator:
+ # Evaluate the MetaModel
+ mean_pred, std_pred = MetaModel.eval_metamodel(samples=theta)
+ else:
+ # Evaluate the origModel
+ mean_pred, std_pred = dict(), dict()
+
+ model_outs, _ = Model.run_model_parallel(
+ theta, prevRun_No=self.counter,
+ key_str='_MCMC', mp=False, verbose=False)
+
+ # Save outputs in respective dicts
+ for varIdx, var in enumerate(Model.Output.names):
+ mean_pred[var] = model_outs[var]
+ std_pred[var] = np.zeros((mean_pred[var].shape))
+
+ # Remove the folder
+ if Model.link_type.lower() != 'function':
+ shutil.rmtree(f"{Model.name}_MCMC_{self.counter+1}")
+
+ # Add one to the counter
+ self.counter += 1
+
+ if hasattr(self, 'error_MetaModel') and BayesObj.error_model:
+ meanPred, stdPred = self.error_MetaModel.eval_model_error(
+ BayesObj.BiasInputs, mean_pred
+ )
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ def train_error_model(self, sampler):
+ """
+ Trains an error model using a Gaussian Process Regression.
+
+ Parameters
+ ----------
+ sampler : obj
+ emcee sampler.
+
+ Returns
+ -------
+ error_MetaModel : obj
+ A error model.
+
+ """
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.MetaModel
+
+ # Prepare the poster samples
+ try:
+ tau = sampler.get_autocorr_time(tol=0)
+ except emcee.autocorr.AutocorrError:
+ tau = 5
+
+ if all(np.isnan(tau)):
+ tau = 5
+
+ burnin = int(2*np.nanmax(tau))
+ thin = int(0.5*np.nanmin(tau)) if int(0.5*np.nanmin(tau)) != 0 else 1
+ finalsamples = sampler.get_chain(discard=burnin, flat=True, thin=thin)
+ posterior = finalsamples[:, :MetaModel.n_params]
+
+ # Select posterior mean as MAP
+ map_theta = posterior.mean(axis=0).reshape((1, MetaModel.n_params))
+ # MAP_theta = st.mode(Posterior_df,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_map, y_std_map = MetaModel.eval_metamodel(samples=map_theta)
+
+ # Train a GPR meta-model using MAP
+ error_MetaModel = MetaModel.create_model_error(
+ BayesObj.BiasInputs, y_map, name='Calib')
+
+ return error_MetaModel
+
+ # -------------------------------------------------------------------------
+ def gelman_rubin(self, chain, return_var=False):
+ """
+ The potential scale reduction factor (PSRF) defined by the variance
+ within one chain, W, with the variance between chains B.
+ Both variances are combined in a weighted sum to obtain an estimate of
+ the variance of a parameter \\( \\theta \\).The square root of the
+ ratio of this estimates variance to the within chain variance is called
+ the potential scale reduction.
+ For a well converged chain it should approach 1. Values greater than
+ 1.1 typically indicate that the chains have not yet fully converged.
+
+ Source: http://joergdietrich.github.io/emcee-convergence.html
+
+ https://github.com/jwalton3141/jwalton3141.github.io/blob/master/assets/posts/ESS/rwmh.py
+
+ Parameters
+ ----------
+ chain : array (n_walkers, n_steps, n_params)
+ The emcee ensamples.
+
+ Returns
+ -------
+ R_hat : float
+ The Gelman-Robin values.
+
+ """
+ m_chains, n_iters = chain.shape[:2]
+
+ # Calculate between-chain variance
+ θb = np.mean(chain, axis=1)
+ θbb = np.mean(θb, axis=0)
+ B_over_n = ((θbb - θb)**2).sum(axis=0)
+ B_over_n /= (m_chains - 1)
+
+ # Calculate within-chain variances
+ ssq = np.var(chain, axis=1, ddof=1)
+ W = np.mean(ssq, axis=0)
+
+ # (over) estimate of variance
+ var_θ = W * (n_iters - 1) / n_iters + B_over_n
+
+ if return_var:
+ return var_θ
+ else:
+ # The square root of the ratio of this estimates variance to the
+ # within chain variance
+ R_hat = np.sqrt(var_θ / W)
+ return R_hat
+
+ # -------------------------------------------------------------------------
+ def marginal_llk_emcee(self, sampler, nburn=None, logp=None, maxiter=1000):
+ """
+ The Bridge Sampling Estimator of the Marginal Likelihood based on
+ https://gist.github.com/junpenglao/4d2669d69ddfe1d788318264cdcf0583
+
+ Parameters
+ ----------
+ sampler : TYPE
+ MultiTrace, result of MCMC run.
+ nburn : int, optional
+ Number of burn-in step. The default is None.
+ logp : TYPE, optional
+ Model Log-probability function. The default is None.
+ maxiter : int, optional
+ Maximum number of iterations. The default is 1000.
+
+ Returns
+ -------
+ marg_llk : dict
+ Estimated Marginal log-Likelihood.
+
+ """
+ r0, tol1, tol2 = 0.5, 1e-10, 1e-4
+
+ if logp is None:
+ logp = sampler.log_prob_fn
+
+ # Split the samples into two parts
+ # Use the first 50% for fiting the proposal distribution
+ # and the second 50% in the iterative scheme.
+ if nburn is None:
+ mtrace = sampler.chain
+ else:
+ mtrace = sampler.chain[:, nburn:, :]
+
+ nchain, len_trace, nrofVars = mtrace.shape
+
+ N1_ = len_trace // 2
+ N1 = N1_*nchain
+ N2 = len_trace*nchain - N1
+
+ samples_4_fit = np.zeros((nrofVars, N1))
+ samples_4_iter = np.zeros((nrofVars, N2))
+ effective_n = np.zeros((nrofVars))
+
+ # matrix with already transformed samples
+ for var in range(nrofVars):
+
+ # for fitting the proposal
+ x = mtrace[:, :N1_, var]
+
+ samples_4_fit[var, :] = x.flatten()
+ # for the iterative scheme
+ x2 = mtrace[:, N1_:, var]
+ samples_4_iter[var, :] = x2.flatten()
+
+ # effective sample size of samples_4_iter, scalar
+ effective_n[var] = self._my_ESS(x2)
+
+ # median effective sample size (scalar)
+ neff = np.median(effective_n)
+
+ # get mean & covariance matrix and generate samples from proposal
+ m = np.mean(samples_4_fit, axis=1)
+ V = np.cov(samples_4_fit)
+ L = chol(V, lower=True)
+
+ # Draw N2 samples from the proposal distribution
+ gen_samples = m[:, None] + np.dot(
+ L, st.norm.rvs(0, 1, size=samples_4_iter.shape)
+ )
+
+ # Evaluate proposal distribution for posterior & generated samples
+ q12 = st.multivariate_normal.logpdf(samples_4_iter.T, m, V)
+ q22 = st.multivariate_normal.logpdf(gen_samples.T, m, V)
+
+ # Evaluate unnormalized posterior for posterior & generated samples
+ q11 = logp(samples_4_iter.T)
+ q21 = logp(gen_samples.T)
+
+ # Run iterative scheme:
+ tmp = self._iterative_scheme(
+ N1, N2, q11, q12, q21, q22, r0, neff, tol1, maxiter, 'r'
+ )
+ if ~np.isfinite(tmp['logml']):
+ warnings.warn(
+ "Logml could not be estimated within maxiter, rerunning with "
+ "adjusted starting value. Estimate might be more variable than"
+ " usual.")
+ # use geometric mean as starting value
+ r0_2 = np.sqrt(tmp['r_vals'][-2]*tmp['r_vals'][-1])
+ tmp = self._iterative_scheme(
+ q11, q12, q21, q22, r0_2, neff, tol2, maxiter, 'logml'
+ )
+
+ marg_llk = dict(
+ logml=tmp['logml'], niter=tmp['niter'], method="normal",
+ q11=q11, q12=q12, q21=q21, q22=q22
+ )
+ return marg_llk
+
+ # -------------------------------------------------------------------------
+ def _iterative_scheme(self, N1, N2, q11, q12, q21, q22, r0, neff, tol,
+ maxiter, criterion):
+ """
+ Iterative scheme as proposed in Meng and Wong (1996) to estimate the
+ marginal likelihood
+
+ """
+ l1 = q11 - q12
+ l2 = q21 - q22
+ # To increase numerical stability,
+ # subtracting the median of l1 from l1 & l2 later
+ lstar = np.median(l1)
+ s1 = neff/(neff + N2)
+ s2 = N2/(neff + N2)
+ r = r0
+ r_vals = [r]
+ logml = np.log(r) + lstar
+ criterion_val = 1 + tol
+
+ i = 0
+ while (i <= maxiter) & (criterion_val > tol):
+ rold = r
+ logmlold = logml
+ numi = np.exp(l2 - lstar)/(s1 * np.exp(l2 - lstar) + s2 * r)
+ deni = 1/(s1 * np.exp(l1 - lstar) + s2 * r)
+ if np.sum(~np.isfinite(numi))+np.sum(~np.isfinite(deni)) > 0:
+ warnings.warn(
+ """Infinite value in iterative scheme, returning NaN.
+ Try rerunning with more samples.""")
+ r = (N1/N2) * np.sum(numi)/np.sum(deni)
+ r_vals.append(r)
+ logml = np.log(r) + lstar
+ i += 1
+ if criterion == 'r':
+ criterion_val = np.abs((r - rold)/r)
+ elif criterion == 'logml':
+ criterion_val = np.abs((logml - logmlold)/logml)
+
+ if i >= maxiter:
+ return dict(logml=np.NaN, niter=i, r_vals=np.asarray(r_vals))
+ else:
+ return dict(logml=logml, niter=i)
+
+ # -------------------------------------------------------------------------
+ def _my_ESS(self, x):
+ """
+ Compute the effective sample size of estimand of interest.
+ Vectorised implementation.
+ https://github.com/jwalton3141/jwalton3141.github.io/blob/master/assets/posts/ESS/rwmh.py
+
+
+ Parameters
+ ----------
+ x : array of shape (n_walkers, n_steps)
+ MCMC Samples.
+
+ Returns
+ -------
+ int
+ Effective sample size.
+
+ """
+ m_chains, n_iters = x.shape
+
+ def variogram(t):
+ variogram = ((x[:, t:] - x[:, :(n_iters - t)])**2).sum()
+ variogram /= (m_chains * (n_iters - t))
+ return variogram
+
+ post_var = self.gelman_rubin(x, return_var=True)
+
+ t = 1
+ rho = np.ones(n_iters)
+ negative_autocorr = False
+
+ # Iterate until the sum of consecutive estimates of autocorrelation is
+ # negative
+ while not negative_autocorr and (t < n_iters):
+ rho[t] = 1 - variogram(t) / (2 * post_var)
+
+ if not t % 2:
+ negative_autocorr = sum(rho[t-1:t+1]) < 0
+
+ t += 1
+
+ return int(m_chains*n_iters / (1 + 2*rho[1:t].sum()))
+
+ # -------------------------------------------------------------------------
+ def _check_ranges(self, theta, ranges):
+ """
+ This function checks if theta lies in the given ranges.
+
+ Parameters
+ ----------
+ theta : array
+ Proposed parameter set.
+ ranges : nested list
+ List of the praremeter ranges.
+
+ Returns
+ -------
+ c : bool
+ If it lies in the given range, it return True else False.
+
+ """
+ c = True
+ # traverse in the list1
+ for i, bounds in enumerate(ranges):
+ x = theta[i]
+ # condition check
+ if x < bounds[0] or x > bounds[1]:
+ c = False
+ return c
+ return c
diff --git a/examples/analytical-function/bayesvalidrox/bayesvalidrox.mplstyle b/examples/analytical-function/bayesvalidrox/bayesvalidrox.mplstyle
new file mode 100644
index 0000000000000000000000000000000000000000..1f31c01f24597de0e0be741be4d3a706c4213a6c
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/bayesvalidrox.mplstyle
@@ -0,0 +1,16 @@
+figure.titlesize : 30
+axes.titlesize : 30
+axes.labelsize : 30
+axes.linewidth : 3
+axes.grid : True
+lines.linewidth : 3
+lines.markersize : 10
+xtick.labelsize : 30
+ytick.labelsize : 30
+legend.fontsize : 30
+font.family : serif
+font.serif : Arial
+font.size : 30
+text.usetex : True
+grid.linestyle : -
+figure.figsize : 24, 16
diff --git a/examples/analytical-function/bayesvalidrox/desktop.ini b/examples/analytical-function/bayesvalidrox/desktop.ini
new file mode 100644
index 0000000000000000000000000000000000000000..632de13ae6b61cecf0d9fdbf9c97cfb16bfb51a4
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/desktop.ini
@@ -0,0 +1,2 @@
+[LocalizedFileNames]
+exploration.py=@exploration.py,0
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__init__.py b/examples/analytical-function/bayesvalidrox/post_processing/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..81c9825420b6ed3f027fb3c141be8af05a89f695
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/post_processing/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .post_processing import PostProcessing
+
+__all__ = [
+ "PostProcessing"
+ ]
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c8590a242166b2e8d40de7ee2eece71980bd1571
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e84acd550fed2f7af8a071adf99001f44547bdf6
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..58a0eb24635d0b97a14d13708e616de6a0659976
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0924d8afac04d4fe82ebe791bc55a8ae48d7c117
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..797c6f82a4d9b29d81d7edc3a3df54e9cf3983aa
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..312575d7655db85df423489051f494f3dce62692
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/post_processing/post_processing.py b/examples/analytical-function/bayesvalidrox/post_processing/post_processing.py
new file mode 100644
index 0000000000000000000000000000000000000000..24ca8d9b9d4ea17268bc395c629c563db46899f4
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/post_processing/post_processing.py
@@ -0,0 +1,1491 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import math
+import os
+from itertools import combinations, cycle
+import pandas as pd
+import scipy.stats as stats
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error, r2_score
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+from matplotlib.offsetbox import AnchoredText
+from matplotlib.patches import Patch
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class PostProcessing:
+ """
+ This class provides many helper functions to post-process the trained
+ meta-model.
+
+ Attributes
+ ----------
+ MetaModel : obj
+ MetaModel object to do postprocessing on.
+ name : str
+ Type of the anaylsis. The default is `'calib'`. If a validation is
+ expected to be performed change this to `'valid'`.
+ out_dir : str
+ Directory to print the results into
+ """
+
+ def __init__(self, engine, name='calib', out_dir = '.'):
+ self.engine = engine
+ self.MetaModel = engine.MetaModel
+ self.ExpDesign = engine.ExpDesign
+ self.ModelObj = engine.Model
+ self.name = name
+ self.out_dir = out_dir
+
+ newpath = (f'Outputs_PostProcessing_{self.name}/')
+ if not os.path.exists(self.out_dir+newpath):
+ os.makedirs(self.out_dir+newpath)
+
+ # -------------------------------------------------------------------------
+
+ def plot_moments(self, xlabel='Time [s]', plot_type=None,
+ use_mc=False, num_mc_samples=1e2):
+ """
+ Plots the moments in a pdf format in the directory
+ `Outputs_PostProcessing`.
+
+ Parameters
+ ----------
+ xlabel : str, optional
+ String to be displayed as x-label. The default is `'Time [s]'`.
+ plot_type : str, optional
+ Options: bar or line. The default is `None`.
+ use_mc : bool, optional
+ Toggles if the moments are calculated from pce coefficients or
+ are estimated from samples
+ num_mc_samples : int, optional
+ The number of samples to use for estimating the moments via MC
+
+ Returns
+ -------
+ pce_means: dict
+ Mean of the model outputs.
+ pce_means: dict
+ Standard deviation of the model outputs.
+
+ """
+
+ bar_plot = True if plot_type == 'bar' else False
+ meta_model_type = self.MetaModel.meta_model_type
+ Model = self.ModelObj
+
+ # Read Monte-Carlo reference
+ self.mc_reference = Model.read_observation('mc_ref')
+
+ # Set the x values
+ x_values_orig = self.engine.ExpDesign.x_values
+
+ # Compute the moments with the PCEModel object
+ # TODO: optimize this call, what is really wanted is if it has polynomial coefficients to use here
+ if self.engine.MetaModel.meta_model_type.lower() == 'gpe' or use_mc == True:
+ self.pce_means, self.pce_stds = self.compute_mc_moments(num_mc_samples)
+ else:
+ self.pce_means, self.pce_stds = self.compute_pce_moments()
+
+ # Get the variables
+ out_names = Model.Output.names
+
+ # Open a pdf for the plots
+ newpath = (f'Outputs_PostProcessing_{self.name}/')
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Plot the best fit line, set the linewidth (lw), color and
+ # transparency (alpha) of the line
+ for key in out_names:
+ fig, ax = plt.subplots(nrows=1, ncols=2)
+
+ # Extract mean and std
+ mean_data = self.pce_means[key]
+ std_data = self.pce_stds[key]
+
+ # Extract a list of x values
+ if type(x_values_orig) is dict:
+ x = x_values_orig[key]
+ else:
+ x = x_values_orig
+
+ # Plot: bar plot or line plot
+ if bar_plot:
+ ax[0].bar(list(map(str, x)), mean_data, color='b',
+ width=0.25)
+ ax[1].bar(list(map(str, x)), std_data, color='b',
+ width=0.25)
+ ax[0].legend(labels=[meta_model_type])
+ ax[1].legend(labels=[meta_model_type])
+ else:
+ ax[0].plot(x, mean_data, lw=3, color='k', marker='x',
+ label=meta_model_type)
+ ax[1].plot(x, std_data, lw=3, color='k', marker='x',
+ label=meta_model_type)
+
+ if self.mc_reference is not None:
+ if bar_plot:
+ ax[0].bar(list(map(str, x)), self.mc_reference['mean'],
+ color='r', width=0.25)
+ ax[1].bar(list(map(str, x)), self.mc_reference['std'],
+ color='r', width=0.25)
+ ax[0].legend(labels=[meta_model_type])
+ ax[1].legend(labels=[meta_model_type])
+ else:
+ ax[0].plot(x, self.mc_reference['mean'], lw=3, marker='x',
+ color='r', label='Ref.')
+ ax[1].plot(x, self.mc_reference['std'], lw=3, marker='x',
+ color='r', label='Ref.')
+
+ # Label the axes and provide a title
+ ax[0].set_xlabel(xlabel)
+ ax[1].set_xlabel(xlabel)
+ ax[0].set_ylabel(key)
+ ax[1].set_ylabel(key)
+
+ # Provide a title
+ ax[0].set_title('Mean of ' + key)
+ ax[1].set_title('Std of ' + key)
+
+ if not bar_plot:
+ ax[0].legend(loc='best')
+ ax[1].legend(loc='best')
+
+ plt.tight_layout()
+
+ # save the current figure
+ fig.savefig(
+ f'{self.out_dir}/{newpath}Mean_Std_PCE_{key}.pdf',
+ bbox_inches='tight'
+ )
+
+ return self.pce_means, self.pce_stds
+
+ # -------------------------------------------------------------------------
+ def valid_metamodel(self, n_samples=1, samples=None, model_out_dict=None,
+ x_axis='Time [s]'):
+ """
+ Evaluates and plots the meta model and the PCEModel outputs for the
+ given number of samples or the given samples.
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of samples to be evaluated. The default is 1.
+ samples : array of shape (n_samples, n_params), optional
+ Samples to be evaluated. The default is None.
+ model_out_dict: dict
+ The model runs using the samples provided.
+ x_axis : str, optional
+ Label of x axis. The default is `'Time [s]'`.
+
+ Returns
+ -------
+ None.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.ModelObj
+
+ if samples is None:
+ self.n_samples = n_samples
+ samples = self._get_sample()
+ else:
+ self.n_samples = samples.shape[0]
+
+ # Extract x_values
+ x_values = self.engine.ExpDesign.x_values
+
+ if model_out_dict is not None:
+ self.model_out_dict = model_out_dict
+ else:
+ self.model_out_dict = self._eval_model(samples, key_str='valid')
+ self.pce_out_mean, self.pce_out_std = MetaModel.eval_metamodel(samples)
+
+ try:
+ key = Model.Output.names[1]
+ except IndexError:
+ key = Model.Output.names[0]
+
+ n_obs = self.model_out_dict[key].shape[1]
+
+ if n_obs == 1:
+ self._plot_validation()
+ else:
+ self._plot_validation_multi(x_values=x_values, x_axis=x_axis)
+
+ # -------------------------------------------------------------------------
+ def check_accuracy(self, n_samples=None, samples=None, outputs=None):
+ """
+ Checks accuracy of the metamodel by computing the root mean square
+ error and validation error for all outputs.
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of samples. The default is None.
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets to be checked. The default is None.
+ outputs : dict, optional
+ Output dictionary with model outputs for all given output types in
+ `Model.Output.names`. The default is None.
+
+ Raises
+ ------
+ Exception
+ When neither n_samples nor samples are provided.
+
+ Returns
+ -------
+ rmse: dict
+ Root mean squared error for each output.
+ valid_error : dict
+ Validation error for each output.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.ModelObj
+
+ # Set the number of samples
+ if n_samples:
+ self.n_samples = n_samples
+ elif samples is not None:
+ self.n_samples = samples.shape[0]
+ else:
+ raise Exception("Please provide either samples or pass the number"
+ " of samples!")
+
+ # Generate random samples if necessary
+ Samples = self._get_sample() if samples is None else samples
+
+ # Run the original model with the generated samples
+ if outputs is None:
+ outputs = self._eval_model(Samples, key_str='validSet')
+
+ # Run the PCE model with the generated samples
+ pce_outputs, _ = MetaModel.eval_metamodel(samples=Samples)
+
+ self.rmse = {}
+ self.valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in Model.Output.names:
+ # Root mean square
+ self.rmse[key] = mean_squared_error(outputs[key], pce_outputs[key],
+ squared=False,
+ multioutput='raw_values')
+ # Validation error
+ self.valid_error[key] = (self.rmse[key]**2) / \
+ np.var(outputs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(self.rmse[key],
+ self.valid_error[key]))))
+ # Save error dicts in PCEModel object
+ self.MetaModel.rmse = self.rmse
+ self.MetaModel.valid_error = self.valid_error
+
+ return
+
+ # -------------------------------------------------------------------------
+ def plot_seq_design_diagnostics(self, ref_BME_KLD=None):
+ """
+ Plots the Bayesian Model Evidence (BME) and Kullback-Leibler divergence
+ (KLD) for the sequential design.
+
+ Parameters
+ ----------
+ ref_BME_KLD : array, optional
+ Reference BME and KLD . The default is `None`.
+
+ Returns
+ -------
+ None.
+
+ """
+ engine = self.engine
+ PCEModel = self.MetaModel
+ n_init_samples = engine.ExpDesign.n_init_samples
+ n_total_samples = engine.ExpDesign.X.shape[0]
+
+ newpath = f'Outputs_PostProcessing_{self.name}/seq_design_diagnostics/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ plotList = ['Modified LOO error', 'Validation error', 'KLD', 'BME',
+ 'RMSEMean', 'RMSEStd', 'Hellinger distance']
+ seqList = [engine.SeqModifiedLOO, engine.seqValidError,
+ engine.SeqKLD, engine.SeqBME, engine.seqRMSEMean,
+ engine.seqRMSEStd, engine.SeqDistHellinger]
+
+ markers = ('x', 'o', 'd', '*', '+')
+ colors = ('k', 'darkgreen', 'b', 'navy', 'darkred')
+
+ # Plot the evolution of the diagnostic criteria of the
+ # Sequential Experimental Design.
+ for plotidx, plot in enumerate(plotList):
+ fig, ax = plt.subplots()
+ seq_dict = seqList[plotidx]
+ name_util = list(seq_dict.keys())
+
+ if len(name_util) == 0:
+ continue
+
+ # Box plot when Replications have been detected.
+ if any(int(name.split("rep_", 1)[1]) > 1 for name in name_util):
+ # Extract the values from dict
+ sorted_seq_opt = {}
+ # Number of replications
+ n_reps = engine.ExpDesign.n_replication
+
+ # Get the list of utility function names
+ # Handle if only one UtilityFunction is provided
+ if not isinstance(engine.ExpDesign.util_func, list):
+ util_funcs = [engine.ExpDesign.util_func]
+ else:
+ util_funcs = engine.ExpDesign.util_func
+
+ for util in util_funcs:
+ sortedSeq = {}
+ # min number of runs available from reps
+ n_runs = min([seq_dict[f'{util}_rep_{i+1}'].shape[0]
+ for i in range(n_reps)])
+
+ for runIdx in range(n_runs):
+ values = []
+ for key in seq_dict.keys():
+ if util in key:
+ values.append(seq_dict[key][runIdx].mean())
+ sortedSeq['SeqItr_'+str(runIdx)] = np.array(values)
+ sorted_seq_opt[util] = sortedSeq
+
+ # BoxPlot
+ def draw_plot(data, labels, edge_color, fill_color, idx):
+ pos = labels - (idx-1)
+ bp = plt.boxplot(data, positions=pos, labels=labels,
+ patch_artist=True, sym='', widths=0.75)
+ elements = ['boxes', 'whiskers', 'fliers', 'means',
+ 'medians', 'caps']
+ for element in elements:
+ plt.setp(bp[element], color=edge_color[idx])
+
+ for patch in bp['boxes']:
+ patch.set(facecolor=fill_color[idx])
+
+ if engine.ExpDesign.n_new_samples != 1:
+ step1 = engine.ExpDesign.n_new_samples
+ step2 = 1
+ else:
+ step1 = 5
+ step2 = 5
+ edge_color = ['red', 'blue', 'green']
+ fill_color = ['tan', 'cyan', 'lightgreen']
+ plot_label = plot
+ # Plot for different Utility Functions
+ for idx, util in enumerate(util_funcs):
+ all_errors = np.empty((n_reps, 0))
+
+ for key in list(sorted_seq_opt[util].keys()):
+ errors = sorted_seq_opt.get(util, {}).get(key)[:, None]
+ all_errors = np.hstack((all_errors, errors))
+
+ # Special cases for BME and KLD
+ if plot == 'KLD' or plot == 'BME':
+ # BME convergence if refBME is provided
+ if ref_BME_KLD is not None:
+ if plot == 'BME':
+ refValue = ref_BME_KLD[0]
+ plot_label = r'BME/BME$^{Ref.}$'
+ if plot == 'KLD':
+ refValue = ref_BME_KLD[1]
+ plot_label = '$D_{KL}[p(\\theta|y_*),p(\\theta)]'\
+ ' / D_{KL}^{Ref.}[p(\\theta|y_*), '\
+ 'p(\\theta)]$'
+
+ # Difference between BME/KLD and the ref. values
+ all_errors = np.divide(all_errors,
+ np.full((all_errors.shape),
+ refValue))
+
+ # Plot baseline for zero, i.e. no difference
+ plt.axhline(y=1.0, xmin=0, xmax=1, c='green',
+ ls='--', lw=2)
+
+ # Plot each UtilFuncs
+ labels = np.arange(n_init_samples, n_total_samples+1, step1)
+ draw_plot(all_errors[:, ::step2], labels, edge_color,
+ fill_color, idx)
+
+ plt.xticks(labels, labels)
+ # Set the major and minor locators
+ ax.xaxis.set_major_locator(ticker.AutoLocator())
+ ax.xaxis.set_minor_locator(ticker.AutoMinorLocator())
+ ax.xaxis.grid(True, which='major', linestyle='-')
+ ax.xaxis.grid(True, which='minor', linestyle='--')
+
+ # Legend
+ legend_elements = []
+ for idx, util in enumerate(util_funcs):
+ legend_elements.append(Patch(facecolor=fill_color[idx],
+ edgecolor=edge_color[idx],
+ label=util))
+ plt.legend(handles=legend_elements[::-1], loc='best')
+
+ if plot != 'BME' and plot != 'KLD':
+ plt.yscale('log')
+ plt.autoscale(True)
+ plt.xlabel('\\# of training samples')
+ plt.ylabel(plot_label)
+ plt.title(plot)
+
+ # save the current figure
+ plot_name = plot.replace(' ', '_')
+ fig.savefig(
+ f'{self.out_dir}/{newpath}/seq_{plot_name}.pdf',
+ bbox_inches='tight'
+ )
+ # Destroy the current plot
+ plt.clf()
+ # Save arrays into files
+ f = open(f'{self.out_dir}/{newpath}/seq_{plot_name}.txt', 'w')
+ f.write(str(sorted_seq_opt))
+ f.close()
+ else:
+ for idx, name in enumerate(name_util):
+ seq_values = seq_dict[name]
+ if engine.ExpDesign.n_new_samples != 1:
+ step = engine.ExpDesign.n_new_samples
+ else:
+ step = 1
+ x_idx = np.arange(n_init_samples, n_total_samples+1, step)
+ if n_total_samples not in x_idx:
+ x_idx = np.hstack((x_idx, n_total_samples))
+
+ if plot == 'KLD' or plot == 'BME':
+ # BME convergence if refBME is provided
+ if ref_BME_KLD is not None:
+ if plot == 'BME':
+ refValue = ref_BME_KLD[0]
+ plot_label = r'BME/BME$^{Ref.}$'
+ if plot == 'KLD':
+ refValue = ref_BME_KLD[1]
+ plot_label = '$D_{KL}[p(\\theta|y_*),p(\\theta)]'\
+ ' / D_{KL}^{Ref.}[p(\\theta|y_*), '\
+ 'p(\\theta)]$'
+
+ # Difference between BME/KLD and the ref. values
+ values = np.divide(seq_values,
+ np.full((seq_values.shape),
+ refValue))
+
+ # Plot baseline for zero, i.e. no difference
+ plt.axhline(y=1.0, xmin=0, xmax=1, c='green',
+ ls='--', lw=2)
+
+ # Set the limits
+ plt.ylim([1e-1, 1e1])
+
+ # Create the plots
+ plt.semilogy(x_idx, values, marker=markers[idx],
+ color=colors[idx], ls='--', lw=2,
+ label=name.split("_rep", 1)[0])
+ else:
+ plot_label = plot
+
+ # Create the plots
+ plt.plot(x_idx, seq_values, marker=markers[idx],
+ color=colors[idx], ls='--', lw=2,
+ label=name.split("_rep", 1)[0])
+
+ else:
+ plot_label = plot
+ seq_values = np.nan_to_num(seq_values)
+
+ # Plot the error evolution for each output
+ plt.semilogy(x_idx, seq_values.mean(axis=1),
+ marker=markers[idx], ls='--', lw=2,
+ color=colors[idx],
+ label=name.split("_rep", 1)[0])
+
+ # Set the major and minor locators
+ ax.xaxis.set_major_locator(ticker.AutoLocator())
+ ax.xaxis.set_minor_locator(ticker.AutoMinorLocator())
+ ax.xaxis.grid(True, which='major', linestyle='-')
+ ax.xaxis.grid(True, which='minor', linestyle='--')
+
+ ax.tick_params(axis='both', which='major', direction='in',
+ width=3, length=10)
+ ax.tick_params(axis='both', which='minor', direction='in',
+ width=2, length=8)
+ plt.xlabel('Number of runs')
+ plt.ylabel(plot_label)
+ plt.title(plot)
+ plt.legend(frameon=True)
+
+ # save the current figure
+ plot_name = plot.replace(' ', '_')
+ fig.savefig(
+ f'{self.out_dir}/{newpath}/seq_{plot_name}.pdf',
+ bbox_inches='tight'
+ )
+ # Destroy the current plot
+ plt.clf()
+
+ # ---------------- Saving arrays into files ---------------
+ np.save(f'{self.out_dir}/{newpath}/seq_{plot_name}.npy', seq_values)
+
+ return
+
+ # -------------------------------------------------------------------------
+ def sobol_indices(self, xlabel='Time [s]', plot_type=None):
+ """
+ Provides Sobol indices as a sensitivity measure to infer the importance
+ of the input parameters. See Eq. 27 in [1] for more details. For the
+ case with Principal component analysis refer to [2].
+
+ [1] Global sensitivity analysis: A flexible and efficient framework
+ with an example from stochastic hydrogeology S. Oladyshkin, F.P.
+ de Barros, W. Nowak https://doi.org/10.1016/j.advwatres.2011.11.001
+
+ [2] Nagel, J.B., Rieckermann, J. and Sudret, B., 2020. Principal
+ component analysis and sparse polynomial chaos expansions for global
+ sensitivity analysis and model calibration: Application to urban
+ drainage simulation. Reliability Engineering & System Safety, 195,
+ p.106737.
+
+ Parameters
+ ----------
+ xlabel : str, optional
+ Label of the x-axis. The default is `'Time [s]'`.
+ plot_type : str, optional
+ Plot type. The default is `None`. This corresponds to line plot.
+ Bar chart can be selected by `bar`.
+
+ Returns
+ -------
+ sobol_cell: dict
+ Sobol indices.
+ total_sobol: dict
+ Total Sobol indices.
+
+ """
+ # Extract the necessary variables
+ PCEModel = self.MetaModel
+ basis_dict = PCEModel.basis_dict
+ coeffs_dict = PCEModel.coeffs_dict
+ n_params = PCEModel.n_params
+ if hasattr(PCEModel, 'n_inner_params'):
+ n_params = PCEModel.n_inner_params
+ max_order = np.max(PCEModel.pce_deg)
+ sobol_cell_b = {}
+ total_sobol_b = {}
+ cov_Z_p_q = np.zeros((n_params))
+
+ outputs = self.ModelObj.Output.names
+ if hasattr(PCEModel, 'trafo_type'):
+ if PCEModel.trafo_type == 'space':
+ outputs = ['Z']
+
+ for b_i in range(PCEModel.n_bootstrap_itrs):
+
+ sobol_cell_, total_sobol_ = {}, {}
+
+ for output in outputs:
+
+ n_meas_points = len(coeffs_dict[f'b_{b_i+1}'][output])
+
+ # Initialize the (cell) array containing the (total) Sobol indices.
+ sobol_array = dict.fromkeys(range(1, max_order+1), [])
+ sobol_cell_array = dict.fromkeys(range(1, max_order+1), [])
+
+ for i_order in range(1, max_order+1):
+ n_comb = math.comb(n_params, i_order)
+
+ sobol_cell_array[i_order] = np.zeros((n_comb, n_meas_points))
+
+ total_sobol_array = np.zeros((n_params, n_meas_points))
+
+ # Initialize the cell to store the names of the variables
+ TotalVariance = np.zeros((n_meas_points))
+ # Loop over all measurement points and calculate sobol indices
+ for pIdx in range(n_meas_points):
+
+ # Extract the basis indices (alpha) and coefficients
+ Basis = basis_dict[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+
+ try:
+ clf_poly = PCEModel.clf_poly[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+ PCECoeffs = clf_poly.coef_
+ except:
+ PCECoeffs = coeffs_dict[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+
+ # Compute total variance
+ TotalVariance[pIdx] = np.sum(np.square(PCECoeffs[1:]))
+
+ nzidx = np.where(PCECoeffs != 0)[0]
+ # Set all the Sobol indices equal to zero in the presence of a
+ # null output.
+ if len(nzidx) == 0:
+ # This is buggy.
+ for i_order in range(1, max_order+1):
+ sobol_cell_array[i_order][:, pIdx] = 0
+
+ # Otherwise compute them by summing well-chosen coefficients
+ else:
+ nz_basis = Basis[nzidx]
+ for i_order in range(1, max_order+1):
+ idx = np.where(np.sum(nz_basis > 0, axis=1) == i_order)
+ subbasis = nz_basis[idx]
+ Z = np.array(list(combinations(range(n_params), i_order)))
+
+ for q in range(Z.shape[0]):
+ Zq = Z[q]
+ subsubbasis = subbasis[:, Zq]
+ subidx = np.prod(subsubbasis, axis=1) > 0
+ sum_ind = nzidx[idx[0][subidx]]
+ if TotalVariance[pIdx] == 0.0:
+ sobol_cell_array[i_order][q, pIdx] = 0.0
+ else:
+ sobol = np.sum(np.square(PCECoeffs[sum_ind]))
+ sobol /= TotalVariance[pIdx]
+ sobol_cell_array[i_order][q, pIdx] = sobol
+
+ # Compute the TOTAL Sobol indices.
+ for ParIdx in range(n_params):
+ idx = nz_basis[:, ParIdx] > 0
+ sum_ind = nzidx[idx]
+
+ if TotalVariance[pIdx] == 0.0:
+ total_sobol_array[ParIdx, pIdx] = 0.0
+ else:
+ sobol = np.sum(np.square(PCECoeffs[sum_ind]))
+ sobol /= TotalVariance[pIdx]
+ total_sobol_array[ParIdx, pIdx] = sobol
+
+ # ----- if PCA selected: Compute covariance -----
+ if PCEModel.dim_red_method.lower() == 'pca':
+ # Extract the basis indices (alpha) and coefficients for
+ # next component
+ if pIdx < n_meas_points-1:
+ nextBasis = basis_dict[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+ if PCEModel.bootstrap_method != 'fast' or b_i == 0:
+ clf_poly = PCEModel.clf_poly[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+ nextPCECoeffs = clf_poly.coef_
+ else:
+ nextPCECoeffs = coeffs_dict[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+
+ # Choose the common non-zero basis
+ mask = (Basis[:, None] == nextBasis).all(-1).any(-1)
+ n_mask = (nextBasis[:, None] == Basis).all(-1).any(-1)
+
+ # Compute the covariance in Eq 17.
+ for ParIdx in range(n_params):
+ idx = (mask) & (Basis[:, ParIdx] > 0)
+ n_idx = (n_mask) & (nextBasis[:, ParIdx] > 0)
+ try:
+ cov_Z_p_q[ParIdx] += np.sum(np.dot(
+ PCECoeffs[idx], nextPCECoeffs[n_idx])
+ )
+ except:
+ pass
+
+ # Compute the sobol indices according to Ref. 2
+ if PCEModel.dim_red_method.lower() == 'pca':
+ n_c_points = self.engine.ExpDesign.Y[output].shape[1]
+ PCA = PCEModel.pca[f'b_{b_i+1}'][output]
+ compPCA = PCA.components_
+ nComp = compPCA.shape[0]
+ var_Z_p = PCA.explained_variance_
+
+ # Extract the sobol index of the components
+ for i_order in range(1, max_order+1):
+ n_comb = math.comb(n_params, i_order)
+ sobol_array[i_order] = np.zeros((n_comb, n_c_points))
+ Z = np.array(list(combinations(range(n_params), i_order)))
+
+ # Loop over parameters
+ for q in range(Z.shape[0]):
+ S_Z_i = sobol_cell_array[i_order][q]
+
+ for tIdx in range(n_c_points):
+ var_Y_t = np.var(
+ self.engine.ExpDesign.Y[output][:, tIdx])
+ if var_Y_t == 0.0:
+ term1, term2 = 0.0, 0.0
+ else:
+ # Eq. 17
+ term1 = 0.0
+ for i in range(nComp):
+ a = S_Z_i[i] * var_Z_p[i]
+ a *= compPCA[i, tIdx]**2
+ term1 += a
+
+ # TODO: Term 2
+ # term2 = 0.0
+ # for i in range(nComp-1):
+ # term2 += cov_Z_p_q[q] * compPCA[i, tIdx]
+ # term2 *= compPCA[i+1, tIdx]
+ # term2 *= 2
+
+ sobol_array[i_order][q, tIdx] = term1 #+ term2
+
+ # Devide over total output variance Eq. 18
+ sobol_array[i_order][q, tIdx] /= var_Y_t
+
+ # Compute the TOTAL Sobol indices.
+ total_sobol = np.zeros((n_params, n_c_points))
+ for ParIdx in range(n_params):
+ S_Z_i = total_sobol_array[ParIdx]
+
+ for tIdx in range(n_c_points):
+ var_Y_t = np.var(self.engine.ExpDesign.Y[output][:, tIdx])
+ if var_Y_t == 0.0:
+ term1, term2 = 0.0, 0.0
+ else:
+ term1 = 0
+ for i in range(nComp):
+ term1 += S_Z_i[i] * var_Z_p[i] * \
+ (compPCA[i, tIdx]**2)
+
+ # Term 2
+ term2 = 0
+ for i in range(nComp-1):
+ term2 += cov_Z_p_q[ParIdx] * compPCA[i, tIdx] \
+ * compPCA[i+1, tIdx]
+ term2 *= 2
+
+ total_sobol[ParIdx, tIdx] = term1 #+ term2
+
+ # Devide over total output variance Eq. 18
+ total_sobol[ParIdx, tIdx] /= var_Y_t
+
+ sobol_cell_[output] = sobol_array
+ total_sobol_[output] = total_sobol
+ else:
+ sobol_cell_[output] = sobol_cell_array
+ total_sobol_[output] = total_sobol_array
+
+ # Save for each bootsrtap iteration
+ sobol_cell_b[b_i] = sobol_cell_
+ total_sobol_b[b_i] = total_sobol_
+
+ # Average total sobol indices
+ total_sobol_all = {}
+ for i in sorted(total_sobol_b):
+ for k, v in total_sobol_b[i].items():
+ if k not in total_sobol_all:
+ total_sobol_all[k] = [None] * len(total_sobol_b)
+ total_sobol_all[k][i] = v
+ sobol_all = {}
+ for i in sorted(sobol_cell_b):
+ for k, v in sobol_cell_b[i].items():
+ for l,m in v.items():
+ if l not in sobol_all:
+ sobol_all[l]={}
+ if k not in sobol_all[l]:
+ sobol_all[l][k] = [None] * len(sobol_cell_b)
+ sobol_all[l][k][i] = v[l]
+ #print(sobol_all)
+
+ self.sobol = {}
+ # Will receive a set of indices for each possible degree of polynomial/interaction
+ for i_order in range(1, max_order+1):
+ self.sobol[i_order]={}
+ for output in outputs:
+ self.sobol[i_order][output] = np.mean([sobol_all[i_order][output]], axis=0)
+
+ self.total_sobol = {}
+ for output in outputs:
+ self.total_sobol[output] = np.mean(total_sobol_all[output], axis=0)
+
+ # ---------------- Plot -----------------------
+ par_names = self.engine.ExpDesign.par_names
+ x_values_orig = self.engine.ExpDesign.x_values
+
+ # Check if the x_values match the number of metamodel outputs
+ if not np.array(x_values_orig).shape[0] == self.total_sobol[outputs[0]].shape[1]:
+ print('The number of MetaModel outputs does not match the x_values'
+ ' specified in ExpDesign. Images are created with '
+ 'equidistant numbers on the x-axis')
+ x_values_orig = np.arange(0,1,self.total_sobol[output].shape[0])
+
+ # Check if it uses a wrapper structure
+ if hasattr(PCEModel, 'trafo_type'):
+ if PCEModel.trafo_type == 'time':
+ par_names.append('time')
+
+ newpath = (f'Outputs_PostProcessing_{self.name}/')
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ if 1:
+ fig = plt.figure()
+
+ for i_order in range(1, max_order+1):
+ # Change labels to combined params sets for higher order indices
+ if i_order == 1:
+ par_names_i = par_names
+ else:
+ par_names_i = list(combinations(par_names, i_order))
+ for outIdx, output in enumerate(outputs):
+
+ # Extract total Sobol indices
+ sobol = self.sobol[i_order][output][0]
+
+ # Compute quantiles
+ q_5 = np.quantile(sobol_all[i_order][output], q=0.05, axis=0)
+ q_97_5 = np.quantile(sobol_all[i_order][output], q=0.975, axis=0)
+
+ # Extract a list of x values
+ if type(x_values_orig) is dict:
+ x = x_values_orig[output]
+ else:
+ x = x_values_orig
+
+ if plot_type == 'bar':
+ ax = fig.add_axes([0, 0, 1, 1])
+ dict1 = {xlabel: x}
+ dict2 = {param: sobolIndices for param, sobolIndices
+ in zip(par_names_i, sobol)}
+
+ df = pd.DataFrame({**dict1, **dict2})
+ df.plot(x=xlabel, y=par_names_i, kind="bar", ax=ax, rot=0,
+ colormap='Dark2', yerr=q_97_5-q_5)
+ ax.set_ylabel('Sobol indices, $S^T$')
+
+ else:
+ for i, sobolIndices in enumerate(sobol):
+ plt.plot(x, sobolIndices, label=par_names_i[i],
+ marker='x', lw=2.5)
+ plt.fill_between(x, q_5[i], q_97_5[i], alpha=0.15)
+
+ plt.ylabel('Sobol indices, $S^T$')
+ plt.xlabel(xlabel)
+
+ plt.title(f'{i_order} degree Sensitivity analysis of {output}')
+ if plot_type != 'bar':
+ plt.legend(loc='best', frameon=True)
+
+ # Save indices
+ np.savetxt(f'{self.out_dir}/{newpath}sobol_{i_order}_' +
+ output.replace('/', '_') + '.csv',
+ sobol.T, delimiter=',',
+ header=','.join(par_names), comments='')
+
+ # save the current figure
+ #print(f'{self.out_dir}/{newpath}Sobol_indices_{i_order}_{output}.pdf')
+ fig.savefig(
+ f'{self.out_dir}/{newpath}Sobol_indices_{i_order}_{output}.pdf',
+ bbox_inches='tight'
+ )
+
+ # Destroy the current plot
+ plt.clf()
+
+ fig = plt.figure()
+
+ for outIdx, output in enumerate(outputs):
+
+ # Extract total Sobol indices
+ total_sobol = self.total_sobol[output]
+
+ # Compute quantiles
+ q_5 = np.quantile(total_sobol_all[output], q=0.05, axis=0)
+ q_97_5 = np.quantile(total_sobol_all[output], q=0.975, axis=0)
+
+ # Extract a list of x values
+ if type(x_values_orig) is dict:
+ x = x_values_orig[output]
+ else:
+ x = x_values_orig
+
+ if plot_type == 'bar':
+ ax = fig.add_axes([0, 0, 1, 1])
+ dict1 = {xlabel: x}
+ dict2 = {param: sobolIndices for param, sobolIndices
+ in zip(par_names, total_sobol)}
+
+ df = pd.DataFrame({**dict1, **dict2})
+ df.plot(x=xlabel, y=par_names, kind="bar", ax=ax, rot=0,
+ colormap='Dark2', yerr=q_97_5-q_5)
+ ax.set_ylabel('Total Sobol indices, $S^T$')
+
+ else:
+ for i, sobolIndices in enumerate(total_sobol):
+ plt.plot(x, sobolIndices, label=par_names[i],
+ marker='x', lw=2.5)
+ plt.fill_between(x, q_5[i], q_97_5[i], alpha=0.15)
+
+ plt.ylabel('Total Sobol indices, $S^T$')
+ plt.xlabel(xlabel)
+
+ plt.title(f'Sensitivity analysis of {output}')
+ if plot_type != 'bar':
+ plt.legend(loc='best', frameon=True)
+
+ # Save indices
+ np.savetxt(f'{self.out_dir}/{newpath}totalsobol_' +
+ output.replace('/', '_') + '.csv',
+ total_sobol.T, delimiter=',',
+ header=','.join(par_names), comments='')
+
+ # save the current figure
+ fig.savefig(
+ f'{self.out_dir}/{newpath}TotalSobol_indices_{output}.pdf',
+ bbox_inches='tight'
+ )
+
+ # Destroy the current plot
+ plt.clf()
+
+ return self.sobol, self.total_sobol
+
+ # -------------------------------------------------------------------------
+ def check_reg_quality(self, n_samples=1000, samples=None):
+ """
+ Checks the quality of the metamodel for single output models based on:
+ https://towardsdatascience.com/how-do-you-check-the-quality-of-your-regression-model-in-python-fa61759ff685
+
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of parameter sets to use for the check. The default is 1000.
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets to use for the check. The default is None.
+
+ Returns
+ -------
+ None.
+
+ """
+ MetaModel = self.MetaModel
+
+ if samples is None:
+ self.n_samples = n_samples
+ samples = self._get_sample()
+ else:
+ self.n_samples = samples.shape[0]
+
+ # Evaluate the original and the surrogate model
+ y_val = self._eval_model(samples, key_str='valid')
+ y_pce_val, _ = MetaModel.eval_metamodel(samples=samples)
+
+ # Open a pdf for the plots
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Fit the data(train the model)
+ for key in y_pce_val.keys():
+
+ y_pce_val_ = y_pce_val[key]
+ y_val_ = y_val[key]
+ residuals = y_val_ - y_pce_val_
+
+ # ------ Residuals vs. predicting variables ------
+ # Check the assumptions of linearity and independence
+ fig1 = plt.figure()
+ for i, par in enumerate(self.engine.ExpDesign.par_names):
+ plt.title(f"{key}: Residuals vs. {par}")
+ plt.scatter(
+ x=samples[:, i], y=residuals, color='blue', edgecolor='k')
+ plt.grid(True)
+ xmin, xmax = min(samples[:, i]), max(samples[:, i])
+ plt.hlines(y=0, xmin=xmin*0.9, xmax=xmax*1.1, color='red',
+ lw=3, linestyle='--')
+ plt.xlabel(par)
+ plt.ylabel('Residuals')
+ plt.show()
+
+ # save the current figure
+ fig1.savefig(f'{self.out_dir}/{newpath}/Residuals_vs_Par_{i+1}.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Fitted vs. residuals ------
+ # Check the assumptions of linearity and independence
+ fig2 = plt.figure()
+ plt.title(f"{key}: Residuals vs. fitted values")
+ plt.scatter(x=y_pce_val_, y=residuals, color='blue', edgecolor='k')
+ plt.grid(True)
+ xmin, xmax = min(y_val_), max(y_val_)
+ plt.hlines(y=0, xmin=xmin*0.9, xmax=xmax*1.1, color='red', lw=3,
+ linestyle='--')
+ plt.xlabel(key)
+ plt.ylabel('Residuals')
+ plt.show()
+
+ # save the current figure
+ fig2.savefig(f'{self.out_dir}/{newpath}/Fitted_vs_Residuals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Histogram of normalized residuals ------
+ fig3 = plt.figure()
+ resid_pearson = residuals / (max(residuals)-min(residuals))
+ plt.hist(resid_pearson, bins=20, edgecolor='k')
+ plt.ylabel('Count')
+ plt.xlabel('Normalized residuals')
+ plt.title(f"{key}: Histogram of normalized residuals")
+
+ # Normality (Shapiro-Wilk) test of the residuals
+ ax = plt.gca()
+ _, p = stats.shapiro(residuals)
+ if p < 0.01:
+ annText = "The residuals seem to come from a Gaussian Process."
+ else:
+ annText = "The normality assumption may not hold."
+ at = AnchoredText(annText, prop=dict(size=30), frameon=True,
+ loc='upper left')
+ at.patch.set_boxstyle("round,pad=0.,rounding_size=0.2")
+ ax.add_artist(at)
+
+ plt.show()
+
+ # save the current figure
+ fig3.savefig(f'{self.out_dir}/{newpath}/Hist_NormResiduals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Q-Q plot of the normalized residuals ------
+ plt.figure()
+ stats.probplot(residuals[:, 0], plot=plt)
+ plt.xticks()
+ plt.yticks()
+ plt.xlabel("Theoretical quantiles")
+ plt.ylabel("Sample quantiles")
+ plt.title(f"{key}: Q-Q plot of normalized residuals")
+ plt.grid(True)
+ plt.show()
+
+ # save the current figure
+ plt.savefig(f'{self.out_dir}/{newpath}/QQPlot_NormResiduals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # -------------------------------------------------------------------------
+ def eval_pce_model_3d(self):
+
+ self.n_samples = 1000
+
+ PCEModel = self.MetaModel
+ Model = self.ModelObj
+ n_samples = self.n_samples
+
+ # Create 3D-Grid
+ # TODO: Make it general
+ x = np.linspace(-5, 10, n_samples)
+ y = np.linspace(0, 15, n_samples)
+
+ X, Y = np.meshgrid(x, y)
+ PCE_Z = np.zeros((self.n_samples, self.n_samples))
+ Model_Z = np.zeros((self.n_samples, self.n_samples))
+
+ for idxMesh in range(self.n_samples):
+ sample_mesh = np.vstack((X[:, idxMesh], Y[:, idxMesh])).T
+
+ univ_p_val = PCEModel.univ_basis_vals(sample_mesh)
+
+ for Outkey, ValuesDict in PCEModel.coeffs_dict.items():
+
+ pce_out_mean = np.zeros((len(sample_mesh), len(ValuesDict)))
+ pce_out_std = np.zeros((len(sample_mesh), len(ValuesDict)))
+ model_outs = np.zeros((len(sample_mesh), len(ValuesDict)))
+
+ for Inkey, InIdxValues in ValuesDict.items():
+ idx = int(Inkey.split('_')[1]) - 1
+ basis_deg_ind = PCEModel.basis_dict[Outkey][Inkey]
+ clf_poly = PCEModel.clf_poly[Outkey][Inkey]
+
+ PSI_Val = PCEModel.create_psi(basis_deg_ind, univ_p_val)
+
+ # Perdiction with error bar
+ y_mean, y_std = clf_poly.predict(PSI_Val, return_std=True)
+
+ pce_out_mean[:, idx] = y_mean
+ pce_out_std[:, idx] = y_std
+
+ # Model evaluation
+ model_out_dict, _ = Model.run_model_parallel(sample_mesh,
+ key_str='Valid3D')
+ model_outs[:, idx] = model_out_dict[Outkey].T
+
+ PCE_Z[:, idxMesh] = y_mean
+ Model_Z[:, idxMesh] = model_outs[:, 0]
+
+ # ---------------- 3D plot for PCEModel -----------------------
+ fig_PCE = plt.figure()
+ ax = plt.axes(projection='3d')
+ ax.plot_surface(X, Y, PCE_Z, rstride=1, cstride=1,
+ cmap='viridis', edgecolor='none')
+ ax.set_title('PCEModel')
+ ax.set_xlabel('$x_1$')
+ ax.set_ylabel('$x_2$')
+ ax.set_zlabel('$f(x_1,x_2)$')
+
+ plt.grid()
+ plt.show()
+
+ # Saving the figure
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # save the figure to file
+ fig_PCE.savefig(f'{self.out_dir}/{newpath}/3DPlot_PCEModel.pdf',
+ bbox_inches='tight')
+ plt.close(fig_PCE)
+
+ # ---------------- 3D plot for Model -----------------------
+ fig_Model = plt.figure()
+ ax = plt.axes(projection='3d')
+ ax.plot_surface(X, Y, PCE_Z, rstride=1, cstride=1,
+ cmap='viridis', edgecolor='none')
+ ax.set_title('Model')
+ ax.set_xlabel('$x_1$')
+ ax.set_ylabel('$x_2$')
+ ax.set_zlabel('$f(x_1,x_2)$')
+
+ plt.grid()
+ plt.show()
+
+ # Save the figure
+ fig_Model.savefig(f'{self.out_dir}/{newpath}/3DPlot_Model.pdf',
+ bbox_inches='tight')
+ plt.close(fig_Model)
+
+ return
+
+ # -------------------------------------------------------------------------
+ def compute_pce_moments(self):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moments (mean) of outpust.
+ pce_means: dict
+ The first moments (mean) of outpust.
+
+ """
+
+ MetaModel = self.MetaModel
+ outputs = self.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = np.sqrt(np.dot(pce_var,
+ PCA.components_**2))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ # Print a report table
+ print("\n>>>>> Moments of {} <<<<<".format(output))
+ print("\nIndex | Mean | Std. deviation")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(pce_means[output],
+ pce_stds[output]))))
+ print('-'*40)
+
+ return pce_means, pce_stds
+
+ # -------------------------------------------------------------------------
+ def compute_mc_moments(self, num_mc_samples):
+ """
+ Computes the first two moments of the metamodel using MC samples.
+
+ Parameters
+ ----------
+ num_mc_samples : int
+ The number of samples to estimate the moments by
+
+ Returns
+ -------
+ pce_means: dict
+ The first moments (mean) of outpust.
+ pce_means: dict
+ The first moments (mean) of outpust.
+
+ """
+ # Get the MC-samples
+ samples = self.engine.ExpDesign.generate_samples(num_mc_samples)
+ print('Random samples are genereated, now starting the evaluations')
+
+ # Run the metamodel on the samples
+ approx, stds = self.engine.MetaModel.eval_metamodel(samples)
+
+ # Estimate the overall mean and std from this
+ pce_means = {}
+ pce_stds = {}
+ for key in approx.keys():
+ if key == 'x_values':
+ continue
+ pce_means[key] = np.mean(approx[key], axis = 0)
+ pce_stds[key] = np.std(approx[key], axis = 0)
+
+ return pce_means, pce_stds
+
+ # -------------------------------------------------------------------------
+ def _get_sample(self, n_samples=None):
+ """
+ Generates random samples taken from the input parameter space.
+
+ Returns
+ -------
+ samples : array of shape (n_samples, n_params)
+ Generated samples.
+
+ """
+ if n_samples is None:
+ n_samples = self.n_samples
+ self.samples = self.ExpDesign.generate_samples(n_samples,
+ sampling_method='random')
+ return self.samples
+
+ # -------------------------------------------------------------------------
+ def _eval_model(self, samples=None, key_str='Valid'):
+ """
+ Evaluates Forward Model for the given number of self.samples or given
+ samples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate the model at. The default is None.
+ key_str : str, optional
+ Key string pass to the model. The default is 'Valid'.
+
+ Returns
+ -------
+ model_outs : dict
+ Dictionary of results.
+
+ """
+ Model = self.ModelObj
+
+ if samples is None:
+ samples = self._get_sample()
+ self.samples = samples
+ else:
+ self.n_samples = len(samples)
+
+ model_outs, _ = Model.run_model_parallel(samples, key_str=key_str)
+
+ return model_outs
+
+ # -------------------------------------------------------------------------
+ def _plot_validation(self):
+ """
+ Plots outputs for visual comparison of metamodel outputs with that of
+ the (full) original model.
+
+ Returns
+ -------
+ None.
+
+ """
+ PCEModel = self.MetaModel
+
+ # get the samples
+ x_val = self.samples
+ y_pce_val = self.pce_out_mean
+ y_val = self.model_out_dict
+
+ # Open a pdf for the plots
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ fig = plt.figure()
+ # Fit the data(train the model)
+ for key in y_pce_val.keys():
+
+ y_pce_val_ = y_pce_val[key]
+ y_val_ = y_val[key]
+
+ regression_model = LinearRegression()
+ regression_model.fit(y_pce_val_, y_val_)
+
+ # Predict
+ x_new = np.linspace(np.min(y_pce_val_), np.max(y_val_), 100)
+ y_predicted = regression_model.predict(x_new[:, np.newaxis])
+
+ plt.scatter(y_pce_val_, y_val_, color='gold', linewidth=2)
+ plt.plot(x_new, y_predicted, color='k')
+
+ # Calculate the adjusted R_squared and RMSE
+ # the total number of explanatory variables in the model
+ # (not including the constant term)
+ length_list = []
+ for key, value in PCEModel.coeffs_dict['b_1'][key].items():
+ length_list.append(len(value))
+ n_predictors = min(length_list)
+ n_samples = x_val.shape[0]
+
+ R2 = r2_score(y_pce_val_, y_val_)
+ AdjR2 = 1 - (1 - R2) * (n_samples - 1) / \
+ (n_samples - n_predictors - 1)
+ rmse = mean_squared_error(y_pce_val_, y_val_, squared=False)
+
+ plt.annotate(f'RMSE = {rmse:.3f}\n Adjusted $R^2$ = {AdjR2:.3f}',
+ xy=(0.05, 0.85), xycoords='axes fraction')
+
+ plt.ylabel("Original Model")
+ plt.xlabel("PCE Model")
+ plt.grid()
+ plt.show()
+
+ # save the current figure
+ plot_name = key.replace(' ', '_')
+ fig.savefig(f'{self.out_dir}/{newpath}/Model_vs_PCEModel_{plot_name}.pdf',
+ bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ # -------------------------------------------------------------------------
+ def _plot_validation_multi(self, x_values=[], x_axis="x [m]"):
+ """
+ Plots outputs for visual comparison of metamodel outputs with that of
+ the (full) multioutput original model
+
+ Parameters
+ ----------
+ x_values : list or array, optional
+ List of x values. The default is [].
+ x_axis : str, optional
+ Label of the x axis. The default is "x [m]".
+
+ Returns
+ -------
+ None.
+
+ """
+ Model = self.ModelObj
+
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # List of markers and colors
+ color = cycle((['b', 'g', 'r', 'y', 'k']))
+ marker = cycle(('x', 'd', '+', 'o', '*'))
+
+ fig = plt.figure()
+ # Plot the model vs PCE model
+ for keyIdx, key in enumerate(Model.Output.names):
+
+ y_pce_val = self.pce_out_mean[key]
+ y_pce_val_std = self.pce_out_std[key]
+ y_val = self.model_out_dict[key]
+ try:
+ x = self.model_out_dict['x_values'][key]
+ except (TypeError, IndexError):
+ x = x_values
+
+ for idx in range(y_val.shape[0]):
+ Color = next(color)
+ Marker = next(marker)
+
+ plt.plot(x, y_val[idx], color=Color, marker=Marker,
+ label='$Y_{%s}^M$'%(idx+1))
+
+ plt.plot(x, y_pce_val[idx], color=Color, marker=Marker,
+ linestyle='--',
+ label='$Y_{%s}^{PCE}$'%(idx+1))
+ plt.fill_between(x, y_pce_val[idx]-1.96*y_pce_val_std[idx],
+ y_pce_val[idx]+1.96*y_pce_val_std[idx],
+ color=Color, alpha=0.15)
+
+ # Calculate the RMSE
+ rmse = mean_squared_error(y_pce_val, y_val, squared=False)
+ R2 = r2_score(y_pce_val[idx].reshape(-1, 1),
+ y_val[idx].reshape(-1, 1))
+
+ plt.annotate(f'RMSE = {rmse:.3f}\n $R^2$ = {R2:.3f}',
+ xy=(0.85, 0.1), xycoords='axes fraction')
+
+ plt.ylabel(key)
+ plt.xlabel(x_axis)
+ plt.legend(loc='best')
+ plt.grid()
+
+ # save the current figure
+ plot_name = key.replace(' ', '_')
+ fig.savefig(f'{self.out_dir}/{newpath}/Model_vs_PCEModel_{plot_name}.pdf',
+ bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ # Zip the subdirectories
+ Model.zip_subdirs(f'{Model.name}valid', f'{Model.name}valid_')
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__init__.py b/examples/analytical-function/bayesvalidrox/pylink/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bd81739faf43956324b30f6d8e5365b29d55677
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/pylink/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .pylink import PyLinkForwardModel
+
+__all__ = [
+ "PyLinkForwardModel"
+ ]
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5b7c1b3926506fb279b856f55ca6120df31b8888
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1947ad354716d0293953761f0d35193f706cedc1
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0bbb522855ad250ad55bca46123c0f5023076291
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b6ae7c14b35b60388e38fcbd3af64d04771a947c
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6b8b695cb653d82323fc0c075eca519e389c960e
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..746c82eb52be2e437c61bd201433f9d38b8ab177
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/pylink/pylink.py b/examples/analytical-function/bayesvalidrox/pylink/pylink.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ef7d48e293987941d42ff1414efc1d9fc77af65
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/pylink/pylink.py
@@ -0,0 +1,807 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Calls to the model and evaluations
+"""
+
+from dataclasses import dataclass
+
+import os
+import shutil
+import h5py
+import numpy as np
+import time
+import zipfile
+import pandas as pd
+import multiprocessing
+from functools import partial
+import tqdm
+
+#from multiprocessing import get_context
+from multiprocess import get_context
+
+
+
+def within_range(out, minout, maxout):
+ """
+ Checks if all the values in out lie between minout and maxout
+
+ Parameters
+ ----------
+ out : array or list
+ Data to check against range
+ minout : int
+ Lower bound of the range
+ maxout : int
+ Upper bound of the range
+
+ Returns
+ -------
+ inside : bool
+ True if all values in out are in the specified range
+
+ """
+ try:
+ out = np.array(out)
+ except:
+ raise AttributeError('The given values should be a 1D array, but are not')
+ if out.ndim != 1:
+ raise AttributeError('The given values should be a 1D array, but are not')
+
+ if minout > maxout:
+ raise ValueError('The lower and upper bounds do not form a valid range, they might be switched')
+
+ inside = False
+ if (out > minout).all() and (out < maxout).all():
+ inside = True
+ return inside
+
+
+class PyLinkForwardModel(object):
+ """
+ A forward model binder
+
+ This calss serves as a code wrapper. This wrapper allows the execution of
+ a third-party software/solver within the scope of BayesValidRox.
+
+ Attributes
+ ----------
+ link_type : str
+ The type of the wrapper. The default is `'pylink'`. This runs the
+ third-party software or an executable using a shell command with given
+ input files.
+ Second option is `'function'` which assumed that model can be run using
+ a function written separately in a Python script.
+ name : str
+ Name of the model.
+ py_file : str
+ Python file name without `.py` extension to be run for the `'function'`
+ wrapper. Note that the name of the python file and that of the function
+ must be simillar. This function must recieve the parameters in an array
+ of shape `(n_samples, n_params)` and returns a dictionary with the
+ x_values and output arrays for given output names.
+ func_args : dict
+ Additional arguments for the python file. The default is `{}`.
+ shell_command : str
+ Shell command to be executed for the `'pylink'` wrapper.
+ input_file : str or list
+ The input file to be passed to the `'pylink'` wrapper.
+ input_template : str or list
+ A template input file to be passed to the `'pylink'` wrapper. This file
+ must be a copy of `input_file` with `<Xi>` place holder for the input
+ parameters defined using `inputs` class, with i being the number of
+ parameter. The file name ending should include `.tpl` before the actual
+ extension of the input file, for example, `params.tpl.input`.
+ aux_file : str or list
+ The list of auxiliary files needed for the `'pylink'` wrapper.
+ exe_path : str
+ Execution path if you wish to run the model for the `'pylink'` wrapper
+ in another directory. The default is `None`, which corresponds to the
+ currecnt working directory.
+ output_file_names : list of str
+ List of the name of the model output text files for the `'pylink'`
+ wrapper.
+ output_names : list of str
+ List of the model outputs to be used for the analysis.
+ output_parser : str
+ Name of the model parser file (without `.py` extension) that recieves
+ the `output_file_names` and returns a 2d-array with the first row being
+ the x_values, e.g. x coordinates or time and the rest of raws pass the
+ simulation output for each model output defined in `output_names`. Note
+ that again here the name of the file and that of the function must be
+ the same.
+ multi_process: bool
+ Whether the model runs to be executed in parallel for the `'pylink'`
+ wrapper. The default is `True`.
+ n_cpus: int
+ The number of cpus to be used for the parallel model execution for the
+ `'pylink'` wrapper. The default is `None`, which corresponds to all
+ available cpus.
+ meas_file : str
+ The name of the measurement text-based file. This file must contain
+ x_values as the first column and one column for each model output. The
+ default is `None`. Only needed for the Bayesian Inference.
+ meas_file_valid : str
+ The name of the measurement text-based file for the validation. The
+ default is `None`. Only needed for the validation with Bayesian
+ Inference.
+ mc_ref_file : str
+ The name of the text file for the Monte-Carlo reference (mean and
+ standard deviation) values. It must contain `x_values` as the first
+ column, `mean` as the second column and `std` as the third. It can be
+ used to compare the estimated moments using meta-model in the post-
+ processing step. This is only available for one output.
+ obs_dict : dict
+ A dictionary containing the measurement text-based file. It must
+ contain `x_values` as the first item and one item for each model output
+ . The default is `{}`. Only needed for the Bayesian Inference.
+ obs_dict_valid : dict
+ A dictionary containing the validation measurement text-based file. It
+ must contain `x_values` as the first item and one item for each model
+ output. The default is `{}`.
+ mc_ref_dict : dict
+ A dictionary containing the Monte-Carlo reference (mean and standard
+ deviation) values. It must contain `x_values` as the first item and
+ `mean` as the second item and `std` as the third. The default is `{}`.
+ This is only available for one output.
+ """
+
+ # Nested class
+ @dataclass
+ class OutputData(object):
+ parser: str = ""
+ names: list = None
+ file_names: list = None
+
+ def __init__(self, link_type='pylink', name=None, py_file=None,
+ func_args={}, shell_command='', input_file=None,
+ input_template=None, aux_file=None, exe_path='',
+ output_file_names=[], output_names=[], output_parser='',
+ multi_process=True, n_cpus=None, meas_file=None,
+ meas_file_valid=None, mc_ref_file=None, obs_dict={},
+ obs_dict_valid={}, mc_ref_dict={}):
+ self.link_type = link_type
+ self.name = name
+ self.shell_command = shell_command
+ self.py_file = py_file
+ self.func_args = func_args
+ self.input_file = input_file
+ self.input_template = input_template
+ self.aux_file = aux_file
+ self.exe_path = exe_path
+ self.multi_process = multi_process
+ self.n_cpus = n_cpus
+ self.Output = self.OutputData(
+ parser=output_parser,
+ names=output_names,
+ file_names=output_file_names,
+ )
+ self.n_outputs = len(self.Output.names)
+ self.meas_file = meas_file
+ self.meas_file_valid = meas_file_valid
+ self.mc_ref_file = mc_ref_file
+ self.observations = obs_dict
+ self.observations_valid = obs_dict_valid
+ self.mc_reference = mc_ref_dict
+
+ # -------------------------------------------------------------------------
+ def read_observation(self, case='calib'):
+ """
+ Reads/prepare the observation/measurement data for
+ calibration.
+
+ Parameters
+ ----------
+ case : str
+ The type of observation to read in. Can be either 'calib',
+ 'valid' or 'mc_ref'
+
+ Returns
+ -------
+ DataFrame
+ A dataframe with the calibration data.
+
+ """
+ # TOOD: check that what is read in/transformed matches the expected form of data/reference
+ if case.lower() == 'calib':
+ if isinstance(self.observations, dict) and bool(self.observations):
+ self.observations = pd.DataFrame.from_dict(self.observations)
+ elif self.meas_file is not None:
+ file_path = os.path.join(os.getcwd(), self.meas_file)
+ self.observations = pd.read_csv(file_path, delimiter=',')
+ elif isinstance(self.observations, pd.DataFrame):
+ self.observations = self.observations
+ else:
+ raise Exception("Please provide the observation data as a "
+ "dictionary via observations attribute or pass"
+ " the csv-file path to MeasurementFile "
+ "attribute")
+ # Compute the number of observation
+ self.n_obs = self.observations[self.Output.names].notnull().sum().values.sum()
+ return self.observations
+
+ elif case.lower() == 'valid':
+ if isinstance(self.observations_valid, dict) and \
+ bool(self.observations_valid):
+ self.observations_valid = pd.DataFrame.from_dict(self.observations_valid)
+ elif self.meas_file_valid is not None:
+ file_path = os.path.join(os.getcwd(), self.meas_file_valid)
+ self.observations_valid = pd.read_csv(file_path, delimiter=',')
+ elif isinstance(self.observations_valid, pd.DataFrame):
+ self.observations_valid = self.observations_valid
+ else:
+ raise Exception("Please provide the observation data as a "
+ "dictionary via observations attribute or pass"
+ " the csv-file path to MeasurementFile "
+ "attribute")
+ # Compute the number of observation
+ self.n_obs_valid = self.observations_valid[self.Output.names].notnull().sum().values.sum()
+ return self.observations_valid
+
+ elif case.lower() == 'mc_ref':
+ if self.mc_ref_file is None and \
+ isinstance(self.mc_reference, pd.DataFrame):
+ return self.mc_reference
+ elif isinstance(self.mc_reference, dict) and bool(self.mc_reference):
+ self.mc_reference = pd.DataFrame.from_dict(self.mc_reference)
+ elif self.mc_ref_file is not None:
+ file_path = os.path.join(os.getcwd(), self.mc_ref_file)
+ self.mc_reference = pd.read_csv(file_path, delimiter=',')
+ else:
+ self.mc_reference = None
+ return self.mc_reference
+
+
+ # -------------------------------------------------------------------------
+ def read_output(self):
+ """
+ Reads the the parser output file and returns it as an
+ executable function. It is required when the models returns the
+ simulation outputs in csv files.
+
+ Returns
+ -------
+ Output : func
+ Output parser function.
+
+ """
+ output_func_name = self.Output.parser
+
+ output_func = getattr(__import__(output_func_name), output_func_name)
+
+ file_names = []
+ for File in self.Output.file_names:
+ file_names.append(os.path.join(self.exe_path, File))
+ try:
+ output = output_func(self.name, file_names)
+ except TypeError:
+ output = output_func(file_names)
+ return output
+
+ # -------------------------------------------------------------------------
+ def update_input_params(self, new_input_file, param_set):
+ """
+ Finds this pattern with <X1> in the new_input_file and replace it with
+ the new value from the array param_sets.
+
+ Parameters
+ ----------
+ new_input_file : list
+ List of the input files with the adapted names.
+ param_set : array of shape (n_params)
+ Parameter set.
+
+ Returns
+ -------
+ None.
+
+ """
+ NofPa = param_set.shape[0]
+ text_to_search_list = [f'<X{i+1}>' for i in range(NofPa)]
+
+ for filename in new_input_file:
+ # Read in the file
+ with open(filename, 'r') as file:
+ filedata = file.read()
+
+ # Replace the target string
+ for text_to_search, params in zip(text_to_search_list, param_set):
+ filedata = filedata.replace(text_to_search, f'{params:0.4e}')
+
+ # Write the file out again
+ with open(filename, 'w') as file:
+ file.write(filedata)
+
+ # -------------------------------------------------------------------------
+ def run_command(self, command, output_file_names):
+ """
+ Runs the execution command given by the user to run the given model.
+ It checks if the output files have been generated. If yes, the jobe is
+ done and it extracts and returns the requested output(s). Otherwise,
+ it executes the command again.
+
+ Parameters
+ ----------
+ command : str
+ The shell command to be executed.
+ output_file_names : list
+ Name of the output file names.
+
+ Returns
+ -------
+ simulation_outputs : array of shape (n_obs, n_outputs)
+ Simulation outputs.
+
+ """
+
+ # Check if simulation is finished
+ while True:
+ time.sleep(3)
+ files = os.listdir(".")
+ if all(elem in files for elem in output_file_names):
+ break
+ else:
+ # Run command
+ Process = os.system(f'./../{command}')
+ if Process != 0:
+ print('\nMessage 1:')
+ print(f'\tIf the value of \'{Process}\' is a non-zero value'
+ ', then compilation problems occur \n' % Process)
+ os.chdir("..")
+
+ # Read the output
+ simulation_outputs = self.read_output()
+
+ return simulation_outputs
+
+ # -------------------------------------------------------------------------
+ def run_forwardmodel(self, xx):
+ """
+ This function creates subdirectory for the current run and copies the
+ necessary files to this directory and renames them. Next, it executes
+ the given command.
+
+ Parameters
+ ----------
+ xx : tuple
+ A tuple including parameter set, simulation number and key string.
+
+ Returns
+ -------
+ output : array of shape (n_outputs+1, n_obs)
+ An array passed by the output paraser containing the x_values as
+ the first row and the simulations results stored in the the rest of
+ the array.
+
+ """
+ c_points, run_no, key_str = xx
+
+ # Handle if only one imput file is provided
+ if not isinstance(self.input_template, list):
+ self.input_template = [self.input_template]
+ if not isinstance(self.input_file, list):
+ self.input_file = [self.input_file]
+
+ new_input_file = []
+ # Loop over the InputTemplates:
+ for in_temp in self.input_template:
+ if '/' in in_temp:
+ in_temp = in_temp.split('/')[-1]
+ new_input_file.append(in_temp.split('.tpl')[0] + key_str +
+ f"_{run_no+1}" + in_temp.split('.tpl')[1])
+
+ # Create directories
+ newpath = self.name + key_str + f'_{run_no+1}'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Copy the necessary files to the directories
+ print(self.input_template)
+ for in_temp in self.input_template:
+ # Input file(s) of the model
+ shutil.copy2(in_temp, newpath)
+ # Auxiliary file
+ if self.aux_file is not None:
+ shutil.copy2(self.aux_file, newpath) # Auxiliary file
+
+ # Rename the Inputfile and/or auxiliary file
+ os.chdir(newpath)
+ for input_tem, input_file in zip(self.input_template, new_input_file):
+ if '/' in input_tem:
+ input_tem = input_tem.split('/')[-1]
+ os.rename(input_tem, input_file)
+
+ # Update the parametrs in Input file
+ self.update_input_params(new_input_file, c_points)
+
+ # Update the user defined command and the execution path
+ try:
+ new_command = self.shell_command.replace(self.input_file[0],
+ new_input_file[0])
+ new_command = new_command.replace(self.input_file[1],
+ new_input_file[1])
+ except:
+ new_command = self.shell_command.replace(self.input_file[0],
+ new_input_file[0])
+ # Set the exe path if not provided
+ if not bool(self.exe_path):
+ self.exe_path = os.getcwd()
+
+ # Run the model
+ print(new_command)
+ output = self.run_command(new_command, self.Output.file_names)
+
+ return output
+
+ # -------------------------------------------------------------------------
+ def run_model_parallel(self, c_points, prevRun_No=0, key_str='',
+ mp=True, verbose=True):
+ """
+ Runs model simulations. If mp is true (default), then the simulations
+ are started in parallel.
+
+ Parameters
+ ----------
+ c_points : array of shape (n_samples, n_params)
+ Collocation points (training set).
+ prevRun_No : int, optional
+ Previous run number, in case the sequential design is selected.
+ The default is `0`.
+ key_str : str, optional
+ A descriptive string for validation runs. The default is `''`.
+ mp : bool, optional
+ Multiprocessing. The default is `True`.
+ verbose: bool, optional
+ Verbosity. The default is `True`.
+
+ Returns
+ -------
+ all_outputs : dict
+ A dictionary with x values (time step or point id) and all outputs.
+ Each key contains an array of the shape `(n_samples, n_obs)`.
+ new_c_points : array
+ Updated collocation points (training set). If a simulation does not
+ executed successfully, the parameter set is removed.
+
+ """
+
+ # Initilization
+ n_c_points = len(c_points)
+ all_outputs = {}
+
+ # If the link type is UM-Bridge, then no parallel needs to be started from here
+ if self.link_type.lower() == 'umbridge':
+ import umbridge
+ if not hasattr(self, 'x_values'):
+ raise AttributeError('For model type `umbridge` the attribute `x_values` needs to be set for the model!')
+ # Init model
+ #model = umbridge.HTTPModel('http://localhost:4242', 'forward')
+ self.model = umbridge.HTTPModel(self.host, 'forward') # TODO: is this always forward?
+ Function = self.uMBridge_model
+
+ # Extract the function
+ if self.link_type.lower() == 'function':
+ # Prepare the function
+ Function = getattr(__import__(self.py_file), self.py_file)
+ # ---------------------------------------------------------------
+ # -------------- Multiprocessing with Pool Class ----------------
+ # ---------------------------------------------------------------
+ # Start a pool with the number of CPUs
+ if self.n_cpus is None:
+ n_cpus = multiprocessing.cpu_count()
+ else:
+ n_cpus = self.n_cpus
+
+ # Run forward model
+ if n_c_points == 1 or not mp:
+ if n_c_points== 1:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ group_results = Function(c_points, **self.func_args)
+ else:
+ group_results = self.run_forwardmodel(
+ (c_points[0], prevRun_No, key_str)
+ )
+ else:
+ for i in range(c_points.shape[0]):
+ if i == 0:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ group_results = Function(np.array([c_points[0]]), **self.func_args)
+ else:
+ group_results = self.run_forwardmodel(
+ (c_points[0], prevRun_No, key_str)
+ )
+ for key in group_results:
+ if key != 'x_values':
+ group_results[key] = [group_results[key]]
+ else:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ res = Function(np.array([c_points[i]]), **self.func_args)
+ else:
+ res = self.run_forwardmodel(
+ (c_points[i], prevRun_No, key_str)
+ )
+ for key in res:
+ if key != 'x_values':
+ group_results[key].append(res[key])
+
+ for key in group_results:
+ if key != 'x_values':
+ group_results[key]= np.array(group_results[key])
+
+ elif self.multi_process or mp:
+ with get_context('spawn').Pool(n_cpus) as p:
+ #with multiprocessing.Pool(n_cpus) as p:
+
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ imap_var = p.imap(partial(Function, **self.func_args),
+ c_points[:, np.newaxis])
+ else:
+ args = zip(c_points,
+ [prevRun_No+i for i in range(n_c_points)],
+ [key_str]*n_c_points)
+ imap_var = p.imap(self.run_forwardmodel, args)
+
+ if verbose:
+ desc = f'Running forward model {key_str}'
+ group_results = list(tqdm.tqdm(imap_var, total=n_c_points,
+ desc=desc))
+ else:
+ group_results = list(imap_var)
+
+ # Check for NaN
+ for var_i, var in enumerate(self.Output.names):
+ # If results are given as one dictionary
+ if isinstance(group_results, dict):
+ Outputs = np.asarray(group_results[var])
+ # If results are given as list of dictionaries
+ elif isinstance(group_results, list):
+ cnt = 0
+ for item in group_results:
+ #print(var, c_points[cnt],item[var])
+ cnt+=1
+ Outputs = np.asarray([item[var] for item in group_results],
+ dtype=np.float64)
+ NaN_idx = np.unique(np.argwhere(np.isnan(Outputs))[:, 0])
+ new_c_points = np.delete(c_points, NaN_idx, axis=0)
+ all_outputs[var] = np.atleast_2d(
+ np.delete(Outputs, NaN_idx, axis=0)
+ )
+
+ # Print the collocation points whose simulations crashed
+ if len(NaN_idx) != 0:
+ print('\n')
+ print('*'*20)
+ print("\nThe following parameter sets have been removed:\n",
+ c_points[NaN_idx])
+ print("\n")
+ print('*'*20)
+
+ # Save time steps or x-values
+ if isinstance(group_results, dict):
+ all_outputs["x_values"] = group_results["x_values"]
+ elif any(isinstance(i, dict) for i in group_results):
+ all_outputs["x_values"] = group_results[0]["x_values"]
+
+ # Store simulations in a hdf5 file
+ self._store_simulations(
+ c_points, all_outputs, NaN_idx, key_str, prevRun_No
+ )
+
+ return all_outputs, new_c_points
+
+ def uMBridge_model(self, params):
+ """
+ Function that calls a UMBridge model and transforms its output into the
+ shape expected for the surrogate.
+
+ Parameters
+ ----------
+ params : 2d np.array, shape (#samples, #params)
+ The parameter values for which the model is run.
+
+ Returns
+ -------
+ dict
+ The transformed model outputs.
+
+ """
+ # Run the model
+ #out = np.array(model(np.ndarray.tolist(params), {'level':0}))
+ out = np.array(self.model(np.ndarray.tolist(params), self.modelparams))
+
+ # Sort into dict
+ out_dict = {}
+ cnt = 0
+ for key in self.Output.names:
+ # # If needed resort into single-value outputs
+ # if self.output_type == 'single-valued':
+ # if out.shape[1]>1: # TODO: this doesn't fully seem correct??
+ # for i in range(out[:,key]): # TODO: this doesn't fully seem correct??
+ # new_key = key+str(i)
+ # if new_key not in self.Output.names:
+ # self.Output.names.append(new_key)
+ # if i == 0:
+ # self.Ouptut.names.remove(key)
+ # out_dict[new_key] = out[:,cnt,i] # TODO: not sure about this, need to test
+ # else:
+ # out_dict[key] = out[:,cnt]
+ #
+ #
+ # else:
+ out_dict[key] = out[:,cnt]
+ cnt += 1
+
+
+ ## TODO: how to deal with the x-values?
+ #if self.output_type == 'single-valued':
+ # out_dict['x_values'] = [0]
+ #else:
+ # out_dict['x_values'] = np.arange(0,out[:,0].shape[0],1)
+ out_dict['x_values'] = self.x_values
+
+ #return {'T1':out[:,0], 'T2':out[:,1], 'H1':out[:,2], 'H2':out[:,3],
+ # 'x_values':[0]}
+ return out_dict
+
+ # -------------------------------------------------------------------------
+ def _store_simulations(self, c_points, all_outputs, NaN_idx, key_str,
+ prevRun_No):
+ """
+
+
+ Parameters
+ ----------
+ c_points : TYPE
+ DESCRIPTION.
+ all_outputs : TYPE
+ DESCRIPTION.
+ NaN_idx : TYPE
+ DESCRIPTION.
+ key_str : TYPE
+ DESCRIPTION.
+ prevRun_No : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ # Create hdf5 metadata
+ if key_str == '':
+ hdf5file = f'ExpDesign_{self.name}.hdf5'
+ else:
+ hdf5file = f'ValidSet_{self.name}.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # ---------- Save time steps or x-values ----------
+ if not hdf5_exist:
+ if type(all_outputs["x_values"]) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(self.Output.names):
+ grp_x_values.create_dataset(
+ var, data=all_outputs["x_values"][var]
+ )
+ else:
+ file.create_dataset("x_values", data=all_outputs["x_values"])
+
+ # ---------- Save outputs ----------
+ for varIdx, var in enumerate(self.Output.names):
+ if not hdf5_exist:
+ grpY = file.create_group("EDY/"+var)
+ else:
+ grpY = file.get("EDY/"+var)
+
+ if prevRun_No == 0 and key_str == '':
+ grpY.create_dataset(f'init_{key_str}', data=all_outputs[var])
+ else:
+ try:
+ oldEDY = np.array(file[f'EDY/{var}/adaptive_{key_str}'])
+ del file[f'EDY/{var}/adaptive_{key_str}']
+ data = np.vstack((oldEDY, all_outputs[var]))
+ except KeyError:
+ data = all_outputs[var]
+ grpY.create_dataset('adaptive_'+key_str, data=data)
+
+ if prevRun_No == 0 and key_str == '':
+ grpY.create_dataset(f"New_init_{key_str}",
+ data=all_outputs[var])
+ else:
+ try:
+ name = f'EDY/{var}/New_adaptive_{key_str}'
+ oldEDY = np.array(file[name])
+ del file[f'EDY/{var}/New_adaptive_{key_str}']
+ data = np.vstack((oldEDY, all_outputs[var]))
+ except KeyError:
+ data = all_outputs[var]
+ grpY.create_dataset(f'New_adaptive_{key_str}', data=data)
+
+ # ---------- Save CollocationPoints ----------
+ new_c_points = np.delete(c_points, NaN_idx, axis=0)
+ grpX = file.create_group("EDX") if not hdf5_exist else file.get("EDX")
+ if prevRun_No == 0 and key_str == '':
+ grpX.create_dataset("init_"+key_str, data=c_points)
+ if len(NaN_idx) != 0:
+ grpX.create_dataset("New_init_"+key_str, data=new_c_points)
+
+ else:
+ try:
+ name = f'EDX/adaptive_{key_str}'
+ oldCollocationPoints = np.array(file[name])
+ del file[f'EDX/adaptive_{key_str}']
+ data = np.vstack((oldCollocationPoints, new_c_points))
+ except KeyError:
+ data = new_c_points
+ grpX.create_dataset('adaptive_'+key_str, data=data)
+
+ if len(NaN_idx) != 0:
+ try:
+ name = f'EDX/New_adaptive_{key_str}'
+ oldCollocationPoints = np.array(file[name])
+ del file[f'EDX/New_adaptive_{key_str}']
+ data = np.vstack((oldCollocationPoints, new_c_points))
+ except KeyError:
+ data = new_c_points
+ grpX.create_dataset('New_adaptive_'+key_str, data=data)
+
+ # Close h5py file
+ file.close()
+
+ # -------------------------------------------------------------------------
+ def zip_subdirs(self, dir_name, key):
+ """
+ Zips all the files containing the key(word).
+
+ Parameters
+ ----------
+ dir_name : str
+ Directory name.
+ key : str
+ Keyword to search for.
+
+ Returns
+ -------
+ None.
+
+ """
+ # setup file paths variable
+ dir_list = []
+ file_paths = []
+
+ # Read all directory, subdirectories and file lists
+ dir_path = os.getcwd()
+
+ for root, directories, files in os.walk(dir_path):
+ for directory in directories:
+ # Create the full filepath by using os module.
+ if key in directory:
+ folderPath = os.path.join(dir_path, directory)
+ dir_list.append(folderPath)
+
+ # Loop over the identified directories to store the file paths
+ for direct_name in dir_list:
+ for root, directories, files in os.walk(direct_name):
+ for filename in files:
+ # Create the full filepath by using os module.
+ filePath = os.path.join(root, filename)
+ file_paths.append('.'+filePath.split(dir_path)[1])
+
+ # writing files to a zipfile
+ if len(file_paths) != 0:
+ zip_file = zipfile.ZipFile(dir_name+'.zip', 'w')
+ with zip_file:
+ # writing each file one by one
+ for file in file_paths:
+ zip_file.write(file)
+
+ file_paths = [path for path in os.listdir('.') if key in path]
+
+ for path in file_paths:
+ shutil.rmtree(path)
+
+ print("\n")
+ print(f'{dir_name}.zip has been created successfully!\n')
+
+ return
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__init__.py b/examples/analytical-function/bayesvalidrox/surrogate_models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..70bfb20f570464c2907a0a4128f4ed99b6c13736
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .surrogate_models import MetaModel
+
+__all__ = [
+ "MetaModel"
+ ]
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8c10c82287a57ba1e3b4dd428962e57cdfbc5c58
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4b73e63a3096fbc9afc41bae35a3fcc1d7851166
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f1a3fcc2eed66172304cd27ab5fe111ca0198bf5
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2854217e56fecb2456011a91a984951fed9cbcbb
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ffab8b4f67e52a3128aa8740301f958a0d72c502
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..331e387822163df3108bcfc6ee48301f71d7b4c3
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..82737a42dd7351d06b703b3da838031ba95979da
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..92d0cc0e7a0a07123fdfbc2c777d1b9281a43344
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..15917cec53bee82813d19f9029552d6ccba086cf
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..063355b16a397fb5fd89d38daa0d3ca5a8506766
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..129cf9a600a5d44a75d32f91f8b5a3691faff286
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cbfe4d97e8e83ebc276e45ba6e84f514784f1d0b
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d789b4515f89e02f006dafd2c9d85b8e7bea110
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..27878248cbdaa2abf0af9d51d061aa6e2db86f43
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc01f0f01b8bc70d438a3317b87d304883456f15
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e6dfce1df1a397fca74f54ccdfe90908cfd615f1
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e7e0226dc1e28c09b9cd09a610599007b2267e3c
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..076580dd7fa9e11559ef202903d44e00e25b8a26
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..af7431ce432969f095d1c07f429b8129cb5a2def
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3ca9d281b7fd88a0951f52dac6bff569d402740d
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..98964152a4ea29f85f061ea6ec7daa3df7487230
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..16a383b994853cdb226f7b7fb291cdbef789e1f7
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3d5029084d1c61da714440410f73c745713a34cd
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c3f24cf9cb753b59f9e226f828a7c3598ac65a9a
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ddbbbd244be38568b61c4e2bf42cddfabf3fdc2
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ed2b0f6f101965fb42fe059ec79e8084a4b3a9bf
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1532dc02c21ed5bd3cdc6f0b98478d396f56071b
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b0c91ff3182adb1245aa7f20656e8e94336a438c
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a1e26394365e683dbec6da95fbe223ebcee10ef4
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ebbfad9c34b1f9e6c819ea7cf7852af65591444
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b2b3f8f40181637b13ee1156ffa8a03c1ffce8b3
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bd6eb8fcd459fd95ff1b85514f996344b6e4880c
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..543416416ef052c2402c2e9a97976dc4aab22866
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ef8b32284f1c504f9e5aaf483cbaceba9a47185d
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f1ec9479869fec42f363e7f46e07dba2f1c6be5
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..97ed55acc1e800a138ddf489ccea709a8b28f634
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9809c07a1479221297a3af1a9c0efeccc4e6863e
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0843cdf8bd820d9cccfdcde3a1193f3f416ccc10
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..30079dce4bc04802324f720381b872c1a2f64018
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1aff76e3864a194b97534ad92aa7bb35db1038da
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7a3bbc05003eab405d52a934e83053ae50080d25
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..af35ff42f1ea1e8ee19d476f7b51c19199513cde
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b856dfef2c2658af8ecc6b1ab85b99c499a39705
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f46bce593fcfaf95cb1a46ef07808179601d655b
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f44a774b4165e6ff769d8db2f2c13c4dd0cbe8b3
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a6d61c979e164a0fd590b3f925c2d1ac68adb4fc
Binary files /dev/null and b/examples/analytical-function/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc differ
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/adaptPlot.py b/examples/analytical-function/bayesvalidrox/surrogate_models/adaptPlot.py
new file mode 100644
index 0000000000000000000000000000000000000000..102f0373c1086ba4420ada2fb2fc723b78bbd53f
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/adaptPlot.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 13 13:46:24 2020
+
+@author: farid
+"""
+import os
+from sklearn.metrics import mean_squared_error, r2_score
+from itertools import cycle
+from matplotlib.backends.backend_pdf import PdfPages
+import matplotlib.pyplot as plt
+
+
+def adaptPlot(PCEModel, Y_Val, Y_PC_Val, Y_PC_Val_std, x_values=[],
+ plotED=False, SaveFig=True):
+
+ NrofSamples = PCEModel.ExpDesign.n_new_samples
+ initNSamples = PCEModel.ExpDesign.n_init_samples
+ itrNr = 1 + (PCEModel.ExpDesign.X.shape[0] - initNSamples)//NrofSamples
+
+ oldEDY = PCEModel.ExpDesign.Y
+
+ if SaveFig:
+ newpath = 'adaptivePlots'
+ os.makedirs(newpath, exist_ok=True)
+
+ # create a PdfPages object
+ pdf = PdfPages(f'./{newpath}/Model_vs_PCEModel_itr_{itrNr}.pdf')
+
+ # List of markers and colors
+ color = cycle((['b', 'g', 'r', 'y', 'k']))
+ marker = cycle(('x', 'd', '+', 'o', '*'))
+
+ OutNames = list(Y_Val.keys())
+ x_axis = 'Time [s]'
+
+ if len(OutNames) == 1:
+ OutNames.insert(0, x_axis)
+ try:
+ x_values = Y_Val['x_values']
+ except KeyError:
+ x_values = x_values
+
+ fig = plt.figure(figsize=(24, 16))
+
+ # Plot the model vs PCE model
+ for keyIdx, key in enumerate(PCEModel.ModelObj.Output.names):
+ Y_PC_Val_ = Y_PC_Val[key]
+ Y_PC_Val_std_ = Y_PC_Val_std[key]
+ Y_Val_ = Y_Val[key]
+ if Y_Val_.ndim == 1:
+ Y_Val_ = Y_Val_.reshape(1, -1)
+ old_EDY = oldEDY[key]
+ if isinstance(x_values, dict):
+ x = x_values[key]
+ else:
+ x = x_values
+
+ for idx, y in enumerate(Y_Val_):
+ Color = next(color)
+ Marker = next(marker)
+
+ plt.plot(
+ x, y, color=Color, marker=Marker,
+ lw=2.0, label='$Y_{%s}^{M}$'%(idx+itrNr)
+ )
+
+ plt.plot(
+ x, Y_PC_Val_[idx], color=Color, marker=Marker,
+ lw=2.0, linestyle='--', label='$Y_{%s}^{PCE}$'%(idx+itrNr)
+ )
+ plt.fill_between(
+ x, Y_PC_Val_[idx]-1.96*Y_PC_Val_std_[idx],
+ Y_PC_Val_[idx]+1.96*Y_PC_Val_std_[idx], color=Color,
+ alpha=0.15
+ )
+
+ if plotED:
+ for output in old_EDY:
+ plt.plot(x, output, color='grey', alpha=0.1)
+
+ # Calculate the RMSE
+ RMSE = mean_squared_error(Y_PC_Val_, Y_Val_, squared=False)
+ R2 = r2_score(Y_PC_Val_.reshape(-1, 1), Y_Val_.reshape(-1, 1))
+
+ plt.ylabel(key)
+ plt.xlabel(x_axis)
+ plt.title(key)
+
+ ax = fig.axes[0]
+ ax.legend(loc='best', frameon=True)
+ fig.canvas.draw()
+ ax.text(0.65, 0.85,
+ f'RMSE = {round(RMSE, 3)}\n$R^2$ = {round(R2, 3)}',
+ transform=ax.transAxes, color='black',
+ bbox=dict(facecolor='none',
+ edgecolor='black',
+ boxstyle='round,pad=1')
+ )
+ plt.grid()
+
+ if SaveFig:
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+ pdf.close()
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/apoly_construction.py b/examples/analytical-function/bayesvalidrox/surrogate_models/apoly_construction.py
new file mode 100644
index 0000000000000000000000000000000000000000..40830fe8aaa94248df4828c0c49bd4d23e755abd
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/apoly_construction.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import numpy as np
+
+
+def apoly_construction(Data, degree):
+ """
+ Construction of Data-driven Orthonormal Polynomial Basis
+ Author: Dr.-Ing. habil. Sergey Oladyshkin
+ Department of Stochastic Simulation and Safety Research for Hydrosystems
+ Institute for Modelling Hydraulic and Environmental Systems
+ Universitaet Stuttgart, Pfaffenwaldring 5a, 70569 Stuttgart
+ E-mail: Sergey.Oladyshkin@iws.uni-stuttgart.de
+ http://www.iws-ls3.uni-stuttgart.de
+ The current script is based on definition of arbitrary polynomial chaos
+ expansion (aPC), which is presented in the following manuscript:
+ Oladyshkin, S. and W. Nowak. Data-driven uncertainty quantification using
+ the arbitrary polynomial chaos expansion. Reliability Engineering & System
+ Safety, Elsevier, V. 106, P. 179-190, 2012.
+ DOI: 10.1016/j.ress.2012.05.002.
+
+ Parameters
+ ----------
+ Data : array
+ Raw data.
+ degree : int
+ Maximum polynomial degree.
+
+ Returns
+ -------
+ Polynomial : array
+ The coefficients of the univariate orthonormal polynomials.
+
+ """
+ if Data.ndim !=1:
+ raise AttributeError('Data should be a 1D array')
+
+ # Initialization
+ dd = degree + 1
+ nsamples = len(Data)
+
+ # Forward linear transformation (Avoiding numerical issues)
+ MeanOfData = np.mean(Data)
+ Data = Data/MeanOfData
+
+ # Compute raw moments of input data
+ raw_moments = [np.sum(np.power(Data, p))/nsamples for p in range(2*dd+2)]
+
+ # Main Loop for Polynomial with degree up to dd
+ PolyCoeff_NonNorm = np.empty((0, 1))
+ Polynomial = np.zeros((dd+1, dd+1))
+
+ for degree in range(dd+1):
+ Mm = np.zeros((degree+1, degree+1))
+ Vc = np.zeros((degree+1))
+
+ # Define Moments Matrix Mm
+ for i in range(degree+1):
+ for j in range(degree+1):
+ if (i < degree):
+ Mm[i, j] = raw_moments[i+j]
+
+ elif (i == degree) and (j == degree):
+ Mm[i, j] = 1
+
+ # Numerical Optimization for Matrix Solver
+ Mm[i] = Mm[i] / max(abs(Mm[i]))
+
+ # Defenition of Right Hand side ortogonality conditions: Vc
+ for i in range(degree+1):
+ Vc[i] = 1 if i == degree else 0
+
+ # Solution: Coefficients of Non-Normal Orthogonal Polynomial: Vp Eq.(4)
+ try:
+ Vp = np.linalg.solve(Mm, Vc)
+ except:
+ inv_Mm = np.linalg.pinv(Mm)
+ Vp = np.dot(inv_Mm, Vc.T)
+
+ if degree == 0:
+ PolyCoeff_NonNorm = np.append(PolyCoeff_NonNorm, Vp)
+
+ if degree != 0:
+ if degree == 1:
+ zero = [0]
+ else:
+ zero = np.zeros((degree, 1))
+ PolyCoeff_NonNorm = np.hstack((PolyCoeff_NonNorm, zero))
+
+ PolyCoeff_NonNorm = np.vstack((PolyCoeff_NonNorm, Vp))
+
+ if 100*abs(sum(abs(np.dot(Mm, Vp)) - abs(Vc))) > 0.5:
+ print('\n---> Attention: Computational Error too high !')
+ print('\n---> Problem: Convergence of Linear Solver')
+
+ # Original Numerical Normalization of Coefficients with Norm and
+ # orthonormal Basis computation Matrix Storrage
+ # Note: Polynomial(i,j) correspont to coefficient number "j-1"
+ # of polynomial degree "i-1"
+ P_norm = 0
+ for i in range(nsamples):
+ Poly = 0
+ for k in range(degree+1):
+ if degree == 0:
+ Poly += PolyCoeff_NonNorm[k] * (Data[i]**k)
+ else:
+ Poly += PolyCoeff_NonNorm[degree, k] * (Data[i]**k)
+
+ P_norm += Poly**2 / nsamples
+
+ P_norm = np.sqrt(P_norm)
+
+ for k in range(degree+1):
+ if degree == 0:
+ Polynomial[degree, k] = PolyCoeff_NonNorm[k]/P_norm
+ else:
+ Polynomial[degree, k] = PolyCoeff_NonNorm[degree, k]/P_norm
+
+ # Backward linear transformation to the real data space
+ Data *= MeanOfData
+ for k in range(len(Polynomial)):
+ Polynomial[:, k] = Polynomial[:, k] / (MeanOfData**(k))
+
+ return Polynomial
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/bayes_linear.py b/examples/analytical-function/bayesvalidrox/surrogate_models/bayes_linear.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bd827ac0ecc5b3a38116b21767e8a8799593b24
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/bayes_linear.py
@@ -0,0 +1,523 @@
+import numpy as np
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+from sklearn.utils import check_X_y, check_array, as_float_array
+from sklearn.utils.validation import check_is_fitted
+from scipy.linalg import svd
+import warnings
+from sklearn.preprocessing import normalize as f_normalize
+
+
+
+class BayesianLinearRegression(RegressorMixin,LinearModel):
+ '''
+ Superclass for Empirical Bayes and Variational Bayes implementations of
+ Bayesian Linear Regression Model
+ '''
+ def __init__(self, n_iter, tol, fit_intercept,copy_X, verbose):
+ self.n_iter = n_iter
+ self.fit_intercept = fit_intercept
+ self.copy_X = copy_X
+ self.verbose = verbose
+ self.tol = tol
+
+
+ def _check_convergence(self, mu, mu_old):
+ '''
+ Checks convergence of algorithm using changes in mean of posterior
+ distribution of weights
+ '''
+ return np.sum(abs(mu-mu_old)>self.tol) == 0
+
+
+ def _center_data(self,X,y):
+ ''' Centers data'''
+ X = as_float_array(X,copy = self.copy_X)
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype = X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X,axis = 0)
+ y_mean = np.average(y,axis = 0)
+ X -= X_mean
+ y = y - y_mean
+ else:
+ X_mean = np.zeros(X.shape[1],dtype = X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X,y, X_mean, y_mean, X_std
+
+
+ def predict_dist(self,X):
+ '''
+ Calculates mean and variance of predictive distribution for each data
+ point of test set.(Note predictive distribution for each data point is
+ Gaussian, therefore it is uniquely determined by mean and variance)
+
+ Parameters
+ ----------
+ x: array-like of size (n_test_samples, n_features)
+ Set of features for which corresponding responses should be predicted
+
+ Returns
+ -------
+ :list of two numpy arrays [mu_pred, var_pred]
+
+ mu_pred : numpy array of size (n_test_samples,)
+ Mean of predictive distribution
+
+ var_pred: numpy array of size (n_test_samples,)
+ Variance of predictive distribution
+ '''
+ # Note check_array and check_is_fitted are done within self._decision_function(X)
+ mu_pred = self._decision_function(X)
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ return [mu_pred,var_pred]
+
+
+
+
+class EBLinearRegression(BayesianLinearRegression):
+ '''
+ Bayesian Regression with type II maximum likelihood (Empirical Bayes)
+
+ Parameters:
+ -----------
+ n_iter: int, optional (DEFAULT = 300)
+ Maximum number of iterations
+
+ tol: float, optional (DEFAULT = 1e-3)
+ Threshold for convergence
+
+ optimizer: str, optional (DEFAULT = 'fp')
+ Method for optimization , either Expectation Maximization or
+ Fixed Point Gull-MacKay {'em','fp'}. Fixed point iterations are
+ faster, but can be numerically unstable (especially in case of near perfect fit).
+
+ fit_intercept: bool, optional (DEFAULT = True)
+ If True includes bias term in model
+
+ perfect_fit_tol: float (DEAFAULT = 1e-5)
+ Prevents overflow of precision parameters (this is smallest value RSS can have).
+ ( !!! Note if using EM instead of fixed-point, try smaller values
+ of perfect_fit_tol, for better estimates of variance of predictive distribution )
+
+ alpha: float (DEFAULT = 1)
+ Initial value of precision paramter for coefficients ( by default we define
+ very broad distribution )
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied, otherwise will be
+
+ verbose: bool, optional (Default = False)
+ If True at each iteration progress report is printed out
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ intercept_: float
+ Value of bias term (if fit_intercept is False, then intercept_ = 0)
+
+ alpha_ : float
+ Estimated precision of coefficients
+
+ beta_ : float
+ Estimated precision of noise
+
+ eigvals_ : array, shape = (n_features, )
+ Eigenvalues of covariance matrix (from posterior distribution of weights)
+
+ eigvecs_ : array, shape = (n_features, n_featues)
+ Eigenvectors of covariance matrix (from posterior distribution of weights)
+
+ '''
+
+ def __init__(self,n_iter = 300, tol = 1e-3, optimizer = 'fp', fit_intercept = True,
+ normalize=True, perfect_fit_tol = 1e-6, alpha = 1, copy_X = True, verbose = False):
+ super(EBLinearRegression,self).__init__(n_iter, tol, fit_intercept, copy_X, verbose)
+ if optimizer not in ['em','fp']:
+ raise ValueError('Optimizer can be either "em" or "fp" ')
+ self.optimizer = optimizer
+ self.alpha = alpha
+ self.perfect_fit = False
+ self.normalize = True
+ self.scores_ = [np.NINF]
+ self.perfect_fit_tol = perfect_fit_tol
+
+ def _check_convergence(self, mu, mu_old):
+ '''
+ Checks convergence of algorithm using changes in mean of posterior
+ distribution of weights
+ '''
+ return np.sum(abs(mu-mu_old)>self.tol) == 0
+
+
+ def _center_data(self,X,y):
+ ''' Centers data'''
+ X = as_float_array(X,copy = self.copy_X)
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype = X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X, axis=0)
+ X -= X_mean
+ if self.normalize:
+ X, X_std = f_normalize(X, axis=0, copy=False,
+ return_norm=True)
+ else:
+ X_std = np.ones(X.shape[1], dtype=X.dtype)
+ y_mean = np.average(y, axis=0)
+ y = y - y_mean
+ else:
+ X_mean = np.zeros(X.shape[1],dtype = X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X,y, X_mean, y_mean, X_std
+
+ def fit(self, X, y):
+ '''
+ Fits Bayesian Linear Regression using Empirical Bayes
+
+ Parameters
+ ----------
+ X: array-like of size [n_samples,n_features]
+ Matrix of explanatory variables (should not include bias term)
+
+ y: array-like of size [n_features]
+ Vector of dependent variables.
+
+ Returns
+ -------
+ object: self
+ self
+
+ '''
+ # preprocess data
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # precision of noise & and coefficients
+ alpha = self.alpha
+ var_y = np.var(y)
+ # check that variance is non zero !!!
+ if var_y == 0 :
+ beta = 1e-2
+ else:
+ beta = 1. / np.var(y)
+
+ # to speed all further computations save svd decomposition and reuse it later
+ u,d,vt = svd(X, full_matrices = False)
+ Uy = np.dot(u.T,y)
+ dsq = d**2
+ mu = 0
+
+ for i in range(self.n_iter):
+
+ # find mean for posterior of w ( for EM this is E-step)
+ mu_old = mu
+ if n_samples > n_features:
+ mu = vt.T * d/(dsq+alpha/beta)
+ else:
+ # clever use of SVD here , faster for large n_features
+ mu = u * 1./(dsq + alpha/beta)
+ mu = np.dot(X.T,mu)
+ mu = np.dot(mu,Uy)
+
+ # precompute errors, since both methods use it in estimation
+ error = y - np.dot(X,mu)
+ sqdErr = np.sum(error**2)
+
+ if sqdErr / n_samples < self.perfect_fit_tol:
+ self.perfect_fit = True
+ warnings.warn( ('Almost perfect fit!!! Estimated values of variance '
+ 'for predictive distribution are computed using only RSS'))
+ break
+
+ if self.optimizer == "fp":
+ gamma = np.sum(beta*dsq/(beta*dsq + alpha))
+ # use updated mu and gamma parameters to update alpha and beta
+ # !!! made computation numerically stable for perfect fit case
+ alpha = gamma / (np.sum(mu**2) + np.finfo(np.float32).eps )
+ beta = ( n_samples - gamma ) / (sqdErr + np.finfo(np.float32).eps )
+ else:
+ # M-step, update parameters alpha and beta to maximize ML TYPE II
+ eigvals = 1. / (beta * dsq + alpha)
+ alpha = n_features / ( np.sum(mu**2) + np.sum(1/eigvals) )
+ beta = n_samples / ( sqdErr + np.sum(dsq/eigvals) )
+
+ # if converged or exceeded maximum number of iterations => terminate
+ converged = self._check_convergence(mu_old,mu)
+ if self.verbose:
+ print( "Iteration {0} completed".format(i) )
+ if converged is True:
+ print("Algorithm converged after {0} iterations".format(i))
+ if converged or i==self.n_iter -1:
+ break
+ eigvals = 1./(beta * dsq + alpha)
+ self.coef_ = beta*np.dot(vt.T*d*eigvals ,Uy)
+ self._set_intercept(X_mean,y_mean,X_std)
+ self.beta_ = beta
+ self.alpha_ = alpha
+ self.eigvals_ = eigvals
+ self.eigvecs_ = vt.T
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self,X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of predictive
+ distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ '''
+ y_hat = np.dot(X,self.coef_) + self.intercept_
+
+ if return_std:
+ if self.normalize:
+ X = (X - self._x_mean_) / self._x_std
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ std_hat = np.sqrt(var_pred)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+
+# ============================== VBLR =========================================
+
+def gamma_mean(a,b):
+ '''
+ Computes mean of gamma distribution
+
+ Parameters
+ ----------
+ a: float
+ Shape parameter of Gamma distribution
+
+ b: float
+ Rate parameter of Gamma distribution
+
+ Returns
+ -------
+ : float
+ Mean of Gamma distribution
+ '''
+ return float(a) / b
+
+
+
+class VBLinearRegression(BayesianLinearRegression):
+ '''
+ Implements Bayesian Linear Regression using mean-field approximation.
+ Assumes gamma prior on precision parameters of coefficients and noise.
+
+ Parameters:
+ -----------
+ n_iter: int, optional (DEFAULT = 100)
+ Maximum number of iterations for KL minimization
+
+ tol: float, optional (DEFAULT = 1e-3)
+ Convergence threshold
+
+ fit_intercept: bool, optional (DEFAULT = True)
+ If True will use bias term in model fitting
+
+ a: float, optional (Default = 1e-4)
+ Shape parameter of Gamma prior for precision of coefficients
+
+ b: float, optional (Default = 1e-4)
+ Rate parameter of Gamma prior for precision coefficients
+
+ c: float, optional (Default = 1e-4)
+ Shape parameter of Gamma prior for precision of noise
+
+ d: float, optional (Default = 1e-4)
+ Rate parameter of Gamma prior for precision of noise
+
+ verbose: bool, optional (Default = False)
+ If True at each iteration progress report is printed out
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ intercept_: float
+ Value of bias term (if fit_intercept is False, then intercept_ = 0)
+
+ alpha_ : float
+ Mean of precision of coefficients
+
+ beta_ : float
+ Mean of precision of noise
+
+ eigvals_ : array, shape = (n_features, )
+ Eigenvalues of covariance matrix (from posterior distribution of weights)
+
+ eigvecs_ : array, shape = (n_features, n_featues)
+ Eigenvectors of covariance matrix (from posterior distribution of weights)
+
+ '''
+
+ def __init__(self, n_iter = 100, tol =1e-4, fit_intercept = True,
+ a = 1e-4, b = 1e-4, c = 1e-4, d = 1e-4, copy_X = True,
+ verbose = False):
+ super(VBLinearRegression,self).__init__(n_iter, tol, fit_intercept, copy_X,
+ verbose)
+ self.a,self.b = a, b
+ self.c,self.d = c, d
+
+
+ def fit(self,X,y):
+ '''
+ Fits Variational Bayesian Linear Regression Model
+
+ Parameters
+ ----------
+ X: array-like of size [n_samples,n_features]
+ Matrix of explanatory variables (should not include bias term)
+
+ Y: array-like of size [n_features]
+ Vector of dependent variables.
+
+ Returns
+ -------
+ object: self
+ self
+ '''
+ # preprocess data
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # SVD decomposition, done once , reused at each iteration
+ u,D,vt = svd(X, full_matrices = False)
+ dsq = D**2
+ UY = np.dot(u.T,y)
+
+ # some parameters of Gamma distribution have closed form solution
+ a = self.a + 0.5 * n_features
+ c = self.c + 0.5 * n_samples
+ b,d = self.b, self.d
+
+ # initial mean of posterior for coefficients
+ mu = 0
+
+ for i in range(self.n_iter):
+
+ # update parameters of distribution Q(weights)
+ e_beta = gamma_mean(c,d)
+ e_alpha = gamma_mean(a,b)
+ mu_old = np.copy(mu)
+ mu,eigvals = self._posterior_weights(e_beta,e_alpha,UY,dsq,u,vt,D,X)
+
+ # update parameters of distribution Q(precision of weights)
+ b = self.b + 0.5*( np.sum(mu**2) + np.sum(eigvals))
+
+ # update parameters of distribution Q(precision of likelihood)
+ sqderr = np.sum((y - np.dot(X,mu))**2)
+ xsx = np.sum(dsq*eigvals)
+ d = self.d + 0.5*(sqderr + xsx)
+
+ # check convergence
+ converged = self._check_convergence(mu,mu_old)
+ if self.verbose is True:
+ print("Iteration {0} is completed".format(i))
+ if converged is True:
+ print("Algorithm converged after {0} iterations".format(i))
+
+ # terminate if convergence or maximum number of iterations are achieved
+ if converged or i==(self.n_iter-1):
+ break
+
+ # save necessary parameters
+ self.beta_ = gamma_mean(c,d)
+ self.alpha_ = gamma_mean(a,b)
+ self.coef_, self.eigvals_ = self._posterior_weights(self.beta_, self.alpha_, UY,
+ dsq, u, vt, D, X)
+ self._set_intercept(X_mean,y_mean,X_std)
+ self.eigvecs_ = vt.T
+ return self
+
+
+ def _posterior_weights(self, e_beta, e_alpha, UY, dsq, u, vt, d, X):
+ '''
+ Calculates parameters of approximate posterior distribution
+ of weights
+ '''
+ # eigenvalues of covariance matrix
+ sigma = 1./ (e_beta*dsq + e_alpha)
+
+ # mean of approximate posterior distribution
+ n_samples, n_features = X.shape
+ if n_samples > n_features:
+ mu = vt.T * d/(dsq + e_alpha/e_beta)# + np.finfo(np.float64).eps)
+ else:
+ mu = u * 1./(dsq + e_alpha/e_beta)# + np.finfo(np.float64).eps)
+ mu = np.dot(X.T,mu)
+ mu = np.dot(mu,UY)
+ return mu,sigma
+
+ def predict(self,X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of predictive
+ distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ '''
+ x = (X - self._x_mean_) / self._x_std
+ y_hat = np.dot(x,self.coef_) + self._y_mean
+
+ if return_std:
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ std_hat = np.sqrt(var_pred)
+ return y_hat, std_hat
+ else:
+ return y_hat
\ No newline at end of file
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/desktop.ini b/examples/analytical-function/bayesvalidrox/surrogate_models/desktop.ini
new file mode 100644
index 0000000000000000000000000000000000000000..632de13ae6b61cecf0d9fdbf9c97cfb16bfb51a4
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/desktop.ini
@@ -0,0 +1,2 @@
+[LocalizedFileNames]
+exploration.py=@exploration.py,0
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/engine.py b/examples/analytical-function/bayesvalidrox/surrogate_models/engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..387cec5010373a087b01e838aba89404f2069c51
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/engine.py
@@ -0,0 +1,2233 @@
+# -*- coding: utf-8 -*-
+"""
+Engine to train the surrogate
+
+"""
+import copy
+from copy import deepcopy, copy
+import h5py
+import joblib
+import numpy as np
+import os
+
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import pandas as pd
+import sys
+import seaborn as sns
+from joblib import Parallel, delayed
+
+
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+from bayesvalidrox.bayes_inference.discrepancy import Discrepancy
+from .exploration import Exploration
+import pathlib
+
+#from .inputs import Input
+#from .exp_designs import ExpDesigns
+#from .surrogate_models import MetaModel
+#from bayesvalidrox.post_processing.post_processing import PostProcessing
+
+def hellinger_distance(P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ P = np.array(P)
+ Q= np.array(Q)
+
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+
+def logpdf(x, mean, cov):
+ """
+ Computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+def subdomain(Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : int
+ Number of samples to divide the domain for.
+ n_params : int
+ The number of params to build the subdomains for
+
+ Returns
+ -------
+ Subdomains : List of tuples of tuples
+ Each tuple of tuples divides one set of bounds into n_new_samples parts.
+
+ """
+ n_params = len(Bounds)
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+class Engine():
+
+
+ def __init__(self, MetaMod, Model, ExpDes):
+ self.MetaModel = MetaMod
+ self.Model = Model
+ self.ExpDesign = ExpDes
+ self.parallel = False
+ self.trained = False
+
+ def start_engine(self) -> None:
+ """
+ Do all the preparations that need to be run before the actual training
+
+ Returns
+ -------
+ None
+
+ """
+ self.out_names = self.Model.Output.names
+ self.MetaModel.out_names = self.out_names
+
+
+ def train_normal(self, parallel = False, verbose = False, save = False) -> None:
+ """
+ Trains surrogate on static samples only.
+ Samples are taken from the experimental design and the specified
+ model is run on them.
+ Alternatively the samples can be read in from a provided hdf5 file.
+
+
+ Returns
+ -------
+ None
+
+ """
+
+ ExpDesign = self.ExpDesign
+ MetaModel = self.MetaModel
+
+ # Read ExpDesign (training and targets) from the provided hdf5
+ if ExpDesign.hdf5_file is not None:
+ # TODO: need to run 'generate_ED' as well after this or not?
+ ExpDesign.read_from_file(self.out_names)
+ else:
+ # Check if an old hdf5 file exists: if yes, rename it
+ hdf5file = f'ExpDesign_{self.Model.name}.hdf5'
+ if os.path.exists(hdf5file):
+ # os.rename(hdf5file, 'old_'+hdf5file)
+ file = pathlib.Path(hdf5file)
+ file.unlink()
+
+ # Prepare X samples
+ # For training the surrogate use ExpDesign.X_tr, ExpDesign.X is for the model to run on
+ ExpDesign.generate_ED(ExpDesign.n_init_samples,
+ transform=True,
+ max_pce_deg=np.max(MetaModel.pce_deg))
+
+ # Run simulations at X
+ if not hasattr(ExpDesign, 'Y') or ExpDesign.Y is None:
+ print('\n Now the forward model needs to be run!\n')
+ ED_Y, up_ED_X = self.Model.run_model_parallel(ExpDesign.X, mp = parallel)
+ ExpDesign.Y = ED_Y
+ else:
+ # Check if a dict has been passed.
+ if not type(ExpDesign.Y) is dict:
+ raise Exception('Please provide either a dictionary or a hdf5'
+ 'file to ExpDesign.hdf5_file argument.')
+
+ # Separate output dict and x-values
+ if 'x_values' in ExpDesign.Y:
+ ExpDesign.x_values = ExpDesign.Y['x_values']
+ del ExpDesign.Y['x_values']
+ else:
+ print('No x_values are given, this might lead to issues during PostProcessing')
+
+
+ # Fit the surrogate
+ MetaModel.fit(ExpDesign.X, ExpDesign.Y, parallel, verbose)
+
+ # Save what there is to save
+ if save:
+ # Save surrogate
+ with open(f'surrogates/surrogate_{self.Model.name}.pk1', 'wb') as output:
+ joblib.dump(MetaModel, output, 2)
+
+ # Zip the model run directories
+ if self.Model.link_type.lower() == 'pylink' and\
+ self.ExpDesign.sampling_method.lower() != 'user':
+ self.Model.zip_subdirs(self.Model.name, f'{self.Model.name}_')
+
+ # Set that training was done
+ self.trained = True
+
+
+ def train_sequential(self, parallel = False, verbose = False) -> None:
+ """
+ Train the surrogate in a sequential manner.
+ First build and train evereything on the static samples, then iterate
+ choosing more samples and refitting the surrogate on them.
+
+ Returns
+ -------
+ None
+
+ """
+ #self.train_normal(parallel, verbose)
+ self.parallel = parallel
+ self.train_seq_design(parallel, verbose)
+
+
+ # -------------------------------------------------------------------------
+ def eval_metamodel(self, samples=None, nsamples=None,
+ sampling_method='random', return_samples=False):
+ """
+ Evaluates meta-model at the requested samples. One can also generate
+ nsamples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate meta-model at. The default is None.
+ nsamples : int, optional
+ Number of samples to generate, if no `samples` is provided. The
+ default is None.
+ sampling_method : str, optional
+ Type of sampling, if no `samples` is provided. The default is
+ 'random'.
+ return_samples : bool, optional
+ Retun samples, if no `samples` is provided. The default is False.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean of the predictions.
+ std_pred : dict
+ Standard deviatioon of the predictions.
+ """
+ # Generate or transform (if need be) samples
+ if samples is None:
+ # Generate
+ samples = self.ExpDesign.generate_samples(
+ nsamples,
+ sampling_method
+ )
+
+ # Transformation to other space is to be done in the MetaModel
+ # TODO: sort the transformations better
+ mean_pred, std_pred = self.MetaModel.eval_metamodel(samples)
+
+ if return_samples:
+ return mean_pred, std_pred, samples
+ else:
+ return mean_pred, std_pred
+
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self, parallel = False, verbose = False):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ self.parallel = parallel
+
+ # Initialization
+ self.SeqModifiedLOO = {}
+ self.seqValidError = {}
+ self.SeqBME = {}
+ self.SeqKLD = {}
+ self.SeqDistHellinger = {}
+ self.seqRMSEMean = {}
+ self.seqRMSEStd = {}
+ self.seqMinDist = []
+
+ if not hasattr(self.MetaModel, 'valid_samples'):
+ self.ExpDesign.valid_samples = []
+ self.ExpDesign.valid_model_runs = []
+ self.valid_likelihoods = []
+
+ validError = None
+
+
+ # Determine the metamodel type
+ if self.MetaModel.meta_model_type.lower() != 'gpe':
+ pce = True
+ else:
+ pce = False
+ mc_ref = True if bool(self.Model.mc_reference) else False
+ if mc_ref:
+ self.Model.read_observation('mc_ref')
+
+ # Get the parameters
+ max_n_samples = self.ExpDesign.n_max_samples
+ mod_LOO_threshold = self.ExpDesign.mod_LOO_threshold
+ n_canddidate = self.ExpDesign.n_canddidate
+ post_snapshot = self.ExpDesign.post_snapshot
+ n_replication = self.ExpDesign.n_replication
+ util_func = self.ExpDesign.util_func
+ output_name = self.out_names
+
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [self.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ # TODO: recheck the logic in this if statement
+ if (len(self.Model.observations) != 0 or self.Model.meas_file is not None) and hasattr(self.MetaModel, 'Discrepancy'):
+ self.observations = self.Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ # TODO: TotalSigma2 not defined if not in this else???
+ # TODO: no self.observations if in here
+ TotalSigma2 = {}
+
+ # ---------- Initial self.MetaModel ----------
+ if not self.trained:
+ self.train_normal(parallel = parallel, verbose=verbose)
+
+ initMetaModel = deepcopy(self.MetaModel)
+
+ # Validation error if validation set is provided.
+ if self.ExpDesign.valid_model_runs:
+ init_rmse, init_valid_error = self._validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ TotalSigma2 = self.MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self._BME_Calculator(
+ obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = self.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self._error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean:.2f},"
+ f" {init_rmse_std:.2f}")
+
+ # Read the initial experimental design
+ Xinit = self.ExpDesign.X
+ init_n_samples = len(self.ExpDesign.X)
+ initYprev = self.ExpDesign.Y#initMetaModel.ModelOutputDict
+ #self.MetaModel.ModelOutputDict = self.ExpDesign.Y
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ ## Get some initial statistics
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ #prevExpDesign_dict = {}
+ # Can run sequential design multiple times for comparison
+ for repIdx in range(n_replication):
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # util_func: the function to use inside the type of exploitation
+ for util_f in util_func:
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ self.ExpDesign.X = Xinit
+ self.ExpDesign.Y = initYprev
+ self.ExpDesign.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(self.MetaModel)
+ #prevExpDesign_dict[itr_no] = deepcopy(self.ExpDesign)
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel(
+ samples=Xfull[-1].reshape(1, -1))
+ del prevMetaModel_dict[itr_no-1]
+
+ # Optimal Bayesian Design
+ #self.MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.choose_next_sample(TotalSigma2,
+ n_canddidate,
+ util_f)
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ self.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = self.Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(self.ExpDesign, 'adapt_verbose') and \
+ self.ExpDesign.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = self.MetaModel.eval_metamodel(
+ samples=Xnew
+ )
+ adaptPlot(
+ self.MetaModel, Ynew, y_hat, std_hat,
+ plotED=False
+ )
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ self.ExpDesign.Y[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ self.ExpDesign.sampling_method = 'user'
+ self.ExpDesign.X = Xfull
+ #self.ExpDesign.Y = self.MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = self.ExpDesign.Y
+
+ # Pass the new prior as the input
+ # TODO: another look at this - no difference apc to pce to gpe?
+ self.MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ self.MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ self.MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ self.MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ self.train_normal(parallel=False)
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if self.ExpDesign.valid_model_runs:
+ rmse, validError = self._validError(self.MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self._BME_Calculator(obs_data, TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = self.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = self.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self._error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ self.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ self.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ self.SeqBME[strKey] = SeqBME
+ self.SeqKLD[strKey] = SeqKLD
+ if hasattr(self.MetaModel, 'valid_likelihoods') and \
+ self.valid_likelihoods:
+ self.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ self.seqRMSEMean[strKey] = seqRMSEMean
+ self.seqRMSEStd[strKey] = seqRMSEStd
+
+ # return self.MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = self.ExpDesign.X
+ out_dict_y = self.ExpDesign.Y
+ out_names = self.out_names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, y_hat, std, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Get the data
+ obs_data = self.observations
+ # TODO: this should be optimizable to be calculated explicitly
+ if hasattr(self.Model, 'n_obs'):
+ n_obs = self.Model.n_obs
+ else:
+ n_obs = self.n_obs
+ mc_size = 10000
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((mc_size))
+ # print(y_hat)
+ # print(list[y_hat])
+ #print(std)
+ for key in list(y_hat):
+ #print(std[key])
+ cov = np.diag(std[key]**2)
+ #print(y_hat[key], cov)
+ print(key, y_hat[key], std[key])
+ # TODO: added the allow_singular = True here
+ rv = stats.multivariate_normal(mean=y_hat[key], cov=cov,allow_singular = True)
+ Y_MC[key] = rv.rvs(size=mc_size)
+ logPriorLikelihoods += rv.logpdf(Y_MC[key])
+ std_MC[key] = np.zeros((mc_size, y_hat[key].shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self._normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, mc_size)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods), dtype=np.longdouble)#float128)
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = np.nanmean(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self._normpdf(
+ y_hat, std, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ MetaModel = self.MetaModel
+ out_names = self.out_names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = self.ExpDesign.X
+ oldExpDesignY = self.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+ PCE_Model_can = deepcopy(MetaModel)
+ engine_can = deepcopy(self)
+ # Add the candidate to the ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ NewExpDesignY[key] = np.vstack(
+ (oldExpDesignY[key], Y_PC_can[key])
+ )
+
+ engine_can.ExpDesign.sampling_method = 'user'
+ engine_can.ExpDesign.X = NewExpDesignX
+ #engine_can.ModelOutputDict = NewExpDesignY
+ engine_can.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ engine_can.MetaModel.input_obj.poly_coeffs_flag = False
+ engine_can.start_engine()
+ engine_can.train_normal(parallel=False)
+ engine_can.MetaModel.fit(NewExpDesignX, NewExpDesignY)
+# engine_can.train_norm_design(parallel=False)
+
+ # Set the ExpDesign to its original values
+ engine_can.ExpDesign.X = oldExpDesignX
+ engine_can.ModelOutputDict = oldExpDesignY
+ engine_can.ExpDesign.Y = oldExpDesignY
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = engine_can.MetaModel.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = engine_can.MetaModel.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self._normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.longdouble))#float128))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postLikelihoods /= np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ # Evaluate all candidates
+ y_can, std_can = self.MetaModel.eval_metamodel(samples=candidates)
+ # loop through candidates
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="BAL Design"):
+ y_hat = {key: items[idx] for key, items in y_can.items()}
+ std = {key: items[idx] for key, items in std_can.items()}
+
+ # print(y_hat)
+ # print(std)
+ U_J_d[idx] = self.util_BayesianActiveDesign(
+ y_hat, std, sigma2Dict, var)
+
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithm to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"Global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradeoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.ExpDesign.n_init_samples
+ n_max_samples = self.ExpDesign.n_max_samples
+
+ itrNumber = (self.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.ExpDesign.n_init_samples
+ n_max_samples = self.ExpDesign.n_max_samples
+ itrNumber = (self.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def choose_next_sample(self, sigma2=None, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None. # TODO: default is set to DKL, not none
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ Bounds = self.ExpDesign.bound_tuples
+ n_new_samples = self.ExpDesign.n_new_samples
+ explore_method = self.ExpDesign.explore_method
+ exploit_method = self.ExpDesign.exploit_method
+ n_cand_groups = self.ExpDesign.n_cand_groups
+ tradeoff_scheme = self.ExpDesign.tradeoff_scheme
+
+ old_EDX = self.ExpDesign.X
+ old_EDY = self.ExpDesign.Y.copy()
+ ndim = self.ExpDesign.X.shape[1]
+ OutputNames = self.out_names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+ if not hasattr(self.ExpDesign, 'ExploitFunction'):
+ raise AttributeError('Function `ExploitFunction` not given to the ExpDesign, thus cannor run user-defined sequential scheme')
+ # TODO: syntax does not fully match the rest - can test this??
+ Xnew, filteredSamples = self.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+
+ # Dual-Annealing works differently from the rest, so deal with this first
+ # Here exploration and exploitation are performed simulataneously
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ subdomains = subdomain(Bounds, n_new_samples)
+
+ # Multiprocessing
+ if self.parallel:
+ args = []
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+ # Without multiprocessing
+ else:
+ results = []
+ for i in range(n_new_samples):
+ results.append(self.dual_annealing(exploit_method, subdomains[i], sigma2, var, i))
+
+ # New sample
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+ print("\nXnew:\n", Xnew)
+
+ # Computational cost
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"Elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ return Xnew, None
+
+ # Generate needed Exploration class
+ explore = Exploration(self.ExpDesign, n_candidates)
+ explore.w = 100 # * ndim #500 # TODO: where does this value come from?
+
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+
+ # Generate the candidate samples
+ # TODO: here use the sampling method provided by the expdesign?
+ sampling_method = self.ExpDesign.sampling_method
+
+ # TODO: changed this from 'random' for LOOCV
+ if explore_method == 'LOOCV':
+ allCandidates = self.ExpDesign.generate_samples(n_candidates,
+ sampling_method)
+ else:
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = self.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = self.MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(self.ExpDesign, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradeoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+ # Check if all needed properties are set
+ if not hasattr(self.ExpDesign, 'max_func_itr'):
+ raise AttributeError('max_func_itr not given to the experimental design')
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = self.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = self.ExpDesign.generate_samples(
+ n_candidates, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+ # print(candidates)
+ # print(split_cand)
+ if self.parallel:
+ results = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ else:
+ results = []
+ for i in range(n_cand_groups):
+ results.append(self.run_util_func(exploit_method, split_cand[i], i, sigma2, var, X_MC))
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Accumulate the samples
+ finalCandidates = np.concatenate((allCandidates, candidates), axis = 0)
+ finalCandidates = np.unique(finalCandidates, axis = 0)
+
+ # Calculations take into account both exploration and exploitation
+ # samples without duplicates
+ totalScore = np.zeros(finalCandidates.shape[0])
+ #self.totalScore = totalScore
+
+ for cand_idx in range(finalCandidates.shape[0]):
+ # find candidate indices
+ idx1 = np.where(allCandidates == finalCandidates[cand_idx])[0]
+ idx2 = np.where(candidates == finalCandidates[cand_idx])[0]
+
+ # exploration
+ if idx1 != []:
+ idx1 = idx1[0]
+ totalScore[cand_idx] += explore_w * scoreExploration[idx1]
+
+ # exploitation
+ if idx2 != []:
+ idx2 = idx2[0]
+ totalScore[cand_idx] += exploit_w * norm_U_J_d[idx2]
+
+
+ # Total score
+ totalScore = exploit_w * norm_U_J_d
+ totalScore += explore_w * scoreExploration
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ # Changed this from allCandiates to full set of candidates
+ # TODO: still not changed for e.g. 'Voronoi'
+ Xnew = finalCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradeoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # print(UtilMethod)
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = self.MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+ # print(allModifiedLOO.shape)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ # print(totalScore.shape)
+ # print(explore_w)
+ # print(scoreExploration.shape)
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+
+ # TODO: why does it also return None?
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self # TODO: this doesn't fully seem correct?
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = self.out_names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = self.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ # TODO: this is just a patch, need to look at again!
+ Scores = list(self.MetaModel.score_dict['b_1'][OutputName].values())
+ #print(Scores)
+ #print(self.MetaModel.score_dict)
+ #print(self.MetaModel.score_dict.values())
+ #print(self.MetaModel.score_dict['b_1'].values())
+ #print(self.MetaModel.score_dict['b_1'][OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ # TODO: also patched here
+ BasisIndices = self.MetaModel.basis_dict['b_1'][OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = self.MetaModel.univ_basis_vals(oldExpDesignX)
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.MetaModel.univ_basis_vals(candidates)
+ Psi_c = self.MetaModel.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var.lower() == 'd-opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var.lower() == 'a-opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var.lower() == 'k-opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ # print(var.lower())
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def _normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+ """
+ Calculated gaussian likelihood for given y+std based on given obs+sigma
+ # TODO: is this understanding correct?
+
+ Parameters
+ ----------
+ y_hat_pce : dict of 2d np arrays
+ Mean output of the surrogate.
+ std_pce : dict of 2d np arrays
+ Standard deviation output of the surrogate.
+ obs_data : dict of 1d np arrays
+ Observed data.
+ total_sigma2s : pandas dataframe, matches obs_data
+ Estimated uncertainty for the observed data.
+ rmse : dict, optional
+ RMSE values from validation of the surrogate. The default is None.
+
+ Returns
+ -------
+ likelihoods : dict of float
+ The likelihood for each surrogate eval in y_hat_pce compared to the
+ observations (?).
+
+ """
+
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(self.out_names):
+
+ # (Meta)Model Output
+ # print(y_hat_pce[out])
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+ else:
+ tot_sigma2s += np.mean(std_pce[out])**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+
+ # TODO: remove this here
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def _corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = self.ExpDesign.X # valid_samples
+ model_outputs = self.ExpDesign.Y # valid_model_runs
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = self.out_names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def _posteriorPlot(self, posterior, par_names, key):
+ """
+ Plot the posterior of a specific key as a corner plot
+
+ Parameters
+ ----------
+ posterior : 2d np.array
+ Samples of the posterior.
+ par_names : list of strings
+ List of the parameter names.
+ key : string
+ Output key that this posterior belongs to.
+
+ Returns
+ -------
+ figPosterior : corner.corner
+ Plot of the posterior.
+
+ """
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.ExpDesign.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+
+ # -------------------------------------------------------------------------
+ def _BME_Calculator(self, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ Parameters
+ ----------
+ obs_data : dict of 1d np arrays
+ Observed data.
+ sigma2Dict : pandas dataframe, matches obs_data
+ Estimated uncertainty for the observed data.
+ rmse : dict of floats, optional
+ RMSE values for each output-key. The dafault is None.
+
+ Returns
+ -------
+ (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ """
+ # Initializations
+ if hasattr(self, 'valid_likelihoods'):
+ valid_likelihoods = self.valid_likelihoods
+ else:
+ valid_likelihoods = []
+ valid_likelihoods = np.array(valid_likelihoods)
+
+ post_snapshot = self.ExpDesign.post_snapshot
+ if post_snapshot or valid_likelihoods.shape[0] != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = self.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ Y_MC, std_MC = self.MetaModel.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self._normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(self.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or valid_likelihoods:
+ # Hellinger distance
+ valid_likelihoods = np.array(valid_likelihoods)
+ ref_like = np.log(valid_likelihoods[(valid_likelihoods > 0)])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and self.MetaModel.n_params == 2 and not idx % 5:
+ BayesOpts = BayesInference(self)
+
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ # TODO: check with Farid if this first line is how it should be
+ BayesOpts.measured_data = obs_data
+ obs_data = pd.DataFrame(obs_data, columns=self.out_names)
+ BayesOpts.measurement_error = obs_data
+ # TODO: shouldn't the uncertainty be sigma2Dict instead of obs_data?
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def _validError(self):
+ """
+ Evaluate the metamodel on the validation samples and calculate the
+ error against the corresponding model runs
+
+ Returns
+ -------
+ rms_error : dict
+ RMSE for each validation run.
+ valid_error : dict
+ Normed (?)RMSE for each validation run.
+
+ """
+ # Extract the original model with the generated samples
+ valid_model_runs = self.ExpDesign.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ valid_PCE_runs, _ = self.MetaModel.eval_metamodel(samples=self.ExpDesign.valid_samples)
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in self.out_names:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def _error_Mean_Std(self):
+ """
+ Calculates the error in the overall mean and std approximation of the
+ surrogate against the mc-reference provided to the model.
+ This can only be applied to metamodels of polynomial type
+
+ Returns
+ -------
+ RMSE_Mean : float
+ RMSE of the means
+ RMSE_std : float
+ RMSE of the standard deviations
+
+ """
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self.MetaModel._compute_pce_moments()
+
+ # Compute the root mean squared error
+ for output in self.out_names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ self.Model.mc_reference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ self.Model.mc_reference['std'], pce_stds[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/eval_rec_rule.py b/examples/analytical-function/bayesvalidrox/surrogate_models/eval_rec_rule.py
new file mode 100644
index 0000000000000000000000000000000000000000..b583c7eb2ec58d55d19b34130812730d21a12368
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/eval_rec_rule.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+
+
+Based on the implementation in UQLab [1].
+
+References:
+1. S. Marelli, and B. Sudret, UQLab: A framework for uncertainty quantification
+in Matlab, Proc. 2nd Int. Conf. on Vulnerability, Risk Analysis and Management
+(ICVRAM2014), Liverpool, United Kingdom, 2014, 2554-2563.
+
+2. S. Marelli, N. Lüthen, B. Sudret, UQLab user manual – Polynomial chaos
+expansions, Report # UQLab-V1.4-104, Chair of Risk, Safety and Uncertainty
+Quantification, ETH Zurich, Switzerland, 2021.
+
+Author: Farid Mohammadi, M.Sc.
+E-Mail: farid.mohammadi@iws.uni-stuttgart.de
+Department of Hydromechanics and Modelling of Hydrosystems (LH2)
+Institute for Modelling Hydraulic and Environmental Systems (IWS), University
+of Stuttgart, www.iws.uni-stuttgart.de/lh2/
+Pfaffenwaldring 61
+70569 Stuttgart
+
+Created on Fri Jan 14 2022
+"""
+import numpy as np
+from numpy.polynomial.polynomial import polyval
+
+
+def poly_rec_coeffs(n_max, poly_type, params=None):
+ """
+ Computes the recurrence coefficients for classical Wiener-Askey orthogonal
+ polynomials.
+
+ Parameters
+ ----------
+ n_max : int
+ Maximum polynomial degree.
+ poly_type : string
+ Polynomial type.
+ params : list, optional
+ Parameters required for `laguerre` poly type. The default is None.
+
+ Returns
+ -------
+ AB : dict
+ The 3 term recursive coefficients and the applicable ranges.
+
+ """
+
+ if poly_type == 'legendre':
+
+ def an(n):
+ return np.zeros((n+1, 1))
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = np.sqrt(1./(4-i**-2))
+ return sq_bn
+
+ bounds = [-1, 1]
+
+ elif poly_type == 'hermite':
+
+ def an(n):
+ return np.zeros((n+1, 1))
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = np.sqrt(i)
+ return sq_bn
+
+ bounds = [-np.inf, np.inf]
+
+ elif poly_type == 'laguerre':
+
+ def an(n):
+ a = np.zeros((n+1, 1))
+ for i in range(1, n+1):
+ a[i] = 2*n + params[1]
+ return a
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = -np.sqrt(i * (i+params[1]-1))
+ return sq_bn
+
+ bounds = [0, np.inf]
+
+ AB = {'alpha_beta': np.concatenate((an(n_max), sqrt_bn(n_max)), axis=1),
+ 'bounds': bounds}
+
+ return AB
+
+
+def eval_rec_rule(x, max_deg, poly_type):
+ """
+ Evaluates the polynomial that corresponds to the Jacobi matrix defined
+ from the AB.
+
+ Parameters
+ ----------
+ x : array (n_samples)
+ Points where the polynomials are evaluated.
+ max_deg : int
+ Maximum degree.
+ poly_type : string
+ Polynomial type.
+
+ Returns
+ -------
+ values : array of shape (n_samples, max_deg+1)
+ Polynomials corresponding to the Jacobi matrix.
+
+ """
+ AB = poly_rec_coeffs(max_deg, poly_type)
+ AB = AB['alpha_beta']
+
+ values = np.zeros((len(x), AB.shape[0]+1))
+ values[:, 1] = 1 / AB[0, 1]
+
+ for k in range(AB.shape[0]-1):
+ values[:, k+2] = np.multiply((x - AB[k, 0]), values[:, k+1]) - \
+ np.multiply(values[:, k], AB[k, 1])
+ values[:, k+2] = np.divide(values[:, k+2], AB[k+1, 1])
+ return values[:, 1:]
+
+
+def eval_rec_rule_arbitrary(x, max_deg, poly_coeffs):
+ """
+ Evaluates the polynomial at sample array x.
+
+ Parameters
+ ----------
+ x : array (n_samples)
+ Points where the polynomials are evaluated.
+ max_deg : int
+ Maximum degree.
+ poly_coeffs : dict
+ Polynomial coefficients computed based on moments.
+
+ Returns
+ -------
+ values : array of shape (n_samples, max_deg+1)
+ Univariate Polynomials evaluated at samples.
+
+ """
+ values = np.zeros((len(x), max_deg+1))
+
+ for deg in range(max_deg+1):
+ values[:, deg] = polyval(x, poly_coeffs[deg]).T
+
+ return values
+
+
+def eval_univ_basis(x, max_deg, poly_types, apoly_coeffs=None):
+ """
+ Evaluates univariate regressors along input directions.
+
+ Parameters
+ ----------
+ x : array of shape (n_samples, n_params)
+ Training samples.
+ max_deg : int
+ Maximum polynomial degree.
+ poly_types : list of strings
+ List of polynomial types for all parameters.
+ apoly_coeffs : dict , optional
+ Polynomial coefficients computed based on moments. The default is None.
+
+ Returns
+ -------
+ univ_vals : array of shape (n_samples, n_params, max_deg+1)
+ Univariate polynomials for all degrees and parameters evaluated at x.
+
+ """
+ # Initilize the output array
+ n_samples, n_params = x.shape
+ univ_vals = np.zeros((n_samples, n_params, max_deg+1))
+
+ for i in range(n_params):
+
+ if poly_types[i] == 'arbitrary':
+ polycoeffs = apoly_coeffs[f'p_{i+1}']
+ univ_vals[:, i] = eval_rec_rule_arbitrary(x[:, i], max_deg,
+ polycoeffs)
+ else:
+ univ_vals[:, i] = eval_rec_rule(x[:, i], max_deg, poly_types[i])
+
+ return univ_vals
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/exp_designs.py b/examples/analytical-function/bayesvalidrox/surrogate_models/exp_designs.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa03fe17d96fb2c1f19546b7b72fb2fd6dd1c13a
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/exp_designs.py
@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Experimental design with associated sampling methods
+"""
+
+import numpy as np
+import math
+import itertools
+import chaospy
+import scipy.stats as st
+from tqdm import tqdm
+import h5py
+import os
+
+from .apoly_construction import apoly_construction
+from .input_space import InputSpace
+
+# -------------------------------------------------------------------------
+def check_ranges(theta, ranges):
+ """
+ This function checks if theta lies in the given ranges.
+
+ Parameters
+ ----------
+ theta : array
+ Proposed parameter set.
+ ranges : nested list
+ List of the praremeter ranges.
+
+ Returns
+ -------
+ c : bool
+ If it lies in the given range, it return True else False.
+
+ """
+ c = True
+ # traverse in the list1
+ for i, bounds in enumerate(ranges):
+ x = theta[i]
+ # condition check
+ if x < bounds[0] or x > bounds[1]:
+ c = False
+ return c
+ return c
+
+
+class ExpDesigns(InputSpace):
+ """
+ This class generates samples from the prescribed marginals for the model
+ parameters using the `Input` object.
+
+ Attributes
+ ----------
+ Input : obj
+ Input object containing the parameter marginals, i.e. name,
+ distribution type and distribution parameters or available raw data.
+ meta_Model_type : str
+ Type of the meta_Model_type.
+ sampling_method : str
+ Name of the sampling method for the experimental design. The following
+ sampling method are supported:
+
+ * random
+ * latin_hypercube
+ * sobol
+ * halton
+ * hammersley
+ * chebyshev(FT)
+ * grid(FT)
+ * user
+ hdf5_file : str
+ Name of the hdf5 file that contains the experimental design.
+ n_new_samples : int
+ Number of (initial) training points.
+ n_max_samples : int
+ Number of maximum training points.
+ mod_LOO_threshold : float
+ The modified leave-one-out cross validation threshold where the
+ sequential design stops.
+ tradeoff_scheme : str
+ Trade-off scheme to assign weights to the exploration and exploitation
+ scores in the sequential design.
+ n_canddidate : int
+ Number of candidate training sets to calculate the scores for.
+ explore_method : str
+ Type of the exploration method for the sequential design. The following
+ methods are supported:
+
+ * Voronoi
+ * random
+ * latin_hypercube
+ * LOOCV
+ * dual annealing
+ exploit_method : str
+ Type of the exploitation method for the sequential design. The
+ following methods are supported:
+
+ * BayesOptDesign
+ * BayesActDesign
+ * VarOptDesign
+ * alphabetic
+ * Space-filling
+ util_func : str or list
+ The utility function to be specified for the `exploit_method`. For the
+ available utility functions see Note section.
+ n_cand_groups : int
+ Number of candidate groups. Each group of candidate training sets will
+ be evaulated separately in parallel.
+ n_replication : int
+ Number of replications. Only for comparison. The default is 1.
+ post_snapshot : int
+ Whether to plot the posterior in the sequential design. The default is
+ `True`.
+ step_snapshot : int
+ The number of steps to plot the posterior in the sequential design. The
+ default is 1.
+ max_a_post : list or array
+ Maximum a posteriori of the posterior distribution, if known. The
+ default is `[]`.
+ adapt_verbose : bool
+ Whether to plot the model response vs that of metamodel for the new
+ trining point in the sequential design.
+
+ Note
+ ----------
+ The following utiliy functions for the **exploitation** methods are
+ supported:
+
+ #### BayesOptDesign (when data is available)
+ - DKL (Kullback-Leibler Divergence)
+ - DPP (D-Posterior-percision)
+ - APP (A-Posterior-percision)
+
+ #### VarBasedOptDesign -> when data is not available
+ - Entropy (Entropy/MMSE/active learning)
+ - EIGF (Expected Improvement for Global fit)
+ - LOOCV (Leave-one-out Cross Validation)
+
+ #### alphabetic
+ - D-Opt (D-Optimality)
+ - A-Opt (A-Optimality)
+ - K-Opt (K-Optimality)
+ """
+
+ def __init__(self, Input, meta_Model_type='pce',
+ sampling_method='random', hdf5_file=None,
+ n_new_samples=1, n_max_samples=None, mod_LOO_threshold=1e-16,
+ tradeoff_scheme=None, n_canddidate=1, explore_method='random',
+ exploit_method='Space-filling', util_func='Space-filling',
+ n_cand_groups=4, n_replication=1, post_snapshot=False,
+ step_snapshot=1, max_a_post=[], adapt_verbose=False, max_func_itr=1):
+
+ self.InputObj = Input
+ self.meta_Model_type = meta_Model_type
+ self.sampling_method = sampling_method
+ self.hdf5_file = hdf5_file
+ self.n_new_samples = n_new_samples
+ self.n_max_samples = n_max_samples
+ self.mod_LOO_threshold = mod_LOO_threshold
+ self.explore_method = explore_method
+ self.exploit_method = exploit_method
+ self.util_func = util_func
+ self.tradeoff_scheme = tradeoff_scheme
+ self.n_canddidate = n_canddidate
+ self.n_cand_groups = n_cand_groups
+ self.n_replication = n_replication
+ self.post_snapshot = post_snapshot
+ self.step_snapshot = step_snapshot
+ self.max_a_post = max_a_post
+ self.adapt_verbose = adapt_verbose
+ self.max_func_itr = max_func_itr
+
+ # Other
+ self.apce = None
+ self.ndim = None
+
+ # Init
+ self.check_valid_inputs()
+
+ # -------------------------------------------------------------------------
+ def generate_samples(self, n_samples, sampling_method='random',
+ transform=False):
+ """
+ Generates samples with given sampling method
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+ sampling_method : str, optional
+ Sampling method. The default is `'random'`.
+ transform : bool, optional
+ Transformation via an isoprobabilistic transformation method. The
+ default is `False`.
+
+ Returns
+ -------
+ samples: array of shape (n_samples, n_params)
+ Generated samples from defined model input object.
+
+ """
+ try:
+ samples = chaospy.generate_samples(
+ int(n_samples), domain=self.origJDist, rule=sampling_method
+ )
+ except:
+ samples = self.random_sampler(int(n_samples)).T
+
+ return samples.T
+
+
+
+ # -------------------------------------------------------------------------
+ def generate_ED(self, n_samples, transform=False,
+ max_pce_deg=None):
+ """
+ Generates experimental designs (training set) with the given method.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested training points.
+ sampling_method : str, optional
+ Sampling method. The default is `'random'`.
+ transform : bool, optional
+ Isoprobabilistic transformation. The default is `False`.
+ max_pce_deg : int, optional
+ Maximum PCE polynomial degree. The default is `None`.
+
+ Returns
+ -------
+ None
+
+ """
+ if n_samples <0:
+ raise ValueError('A negative number of samples cannot be created. Please provide positive n_samples')
+ n_samples = int(n_samples)
+
+ if not hasattr(self, 'n_init_samples'):
+ self.n_init_samples = n_samples
+
+ # Generate the samples based on requested method
+ self.init_param_space(max_pce_deg)
+
+ sampling_method = self.sampling_method
+ # Pass user-defined samples as ED
+ if sampling_method == 'user':
+ if not hasattr(self, 'X'):
+ raise AttributeError('User-defined sampling cannot proceed as no samples provided. Please add them to this class as attribute X')
+ if not self.X.ndim == 2:
+ raise AttributeError('The provided samples shuld have 2 dimensions')
+ samples = self.X
+ self.n_samples = len(samples)
+
+ # Sample the distribution of parameters
+ elif self.input_data_given:
+ # Case II: Input values are directly given by the user.
+
+ if sampling_method == 'random':
+ samples = self.random_sampler(n_samples)
+
+ elif sampling_method == 'PCM' or \
+ sampling_method == 'LSCM':
+ samples = self.pcm_sampler(n_samples, max_pce_deg)
+
+ else:
+ # Create ExpDesign in the actual space using chaospy
+ try:
+ samples = chaospy.generate_samples(n_samples,
+ domain=self.JDist,
+ rule=sampling_method).T
+ except:
+ samples = self.JDist.resample(n_samples).T
+
+ elif not self.input_data_given:
+ # Case I = User passed known distributions
+ samples = chaospy.generate_samples(n_samples, domain=self.JDist,
+ rule=sampling_method).T
+
+ self.X = samples
+
+ def read_from_file(self, out_names):
+ """
+ Reads in the ExpDesign from a provided h5py file and saves the results.
+
+ Parameters
+ ----------
+ out_names : list of strings
+ The keys that are in the outputs (y) saved in the provided file.
+
+ Returns
+ -------
+ None.
+
+ """
+ if self.hdf5_file == None:
+ raise AttributeError('ExpDesign cannot be read in, please provide hdf5 file first')
+
+ # Read hdf5 file
+ f = h5py.File(self.hdf5_file, 'r+')
+
+ # Read EDX and pass it to ExpDesign object
+ try:
+ self.X = np.array(f["EDX/New_init_"])
+ except KeyError:
+ self.X = np.array(f["EDX/init_"])
+
+ # Update number of initial samples
+ self.n_init_samples = self.X.shape[0]
+
+ # Read EDX and pass it to ExpDesign object
+ self.Y = {}
+
+ # Extract x values
+ try:
+ self.Y["x_values"] = dict()
+ for varIdx, var in enumerate(out_names):
+ x = np.array(f[f"x_values/{var}"])
+ self.Y["x_values"][var] = x
+ except KeyError:
+ self.Y["x_values"] = np.array(f["x_values"])
+
+ # Store the output
+ for varIdx, var in enumerate(out_names):
+ try:
+ y = np.array(f[f"EDY/{var}/New_init_"])
+ except KeyError:
+ y = np.array(f[f"EDY/{var}/init_"])
+ self.Y[var] = y
+ f.close()
+ print(f'Experimental Design is read in from file {self.hdf5_file}')
+ print('')
+
+
+
+ # -------------------------------------------------------------------------
+ def random_sampler(self, n_samples, max_deg = None):
+ """
+ Samples the given raw data randomly.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+
+ max_deg : int, optional
+ Maximum degree. The default is `None`.
+ This will be used to run init_param_space, if it has not been done
+ until now.
+
+ Returns
+ -------
+ samples: array of shape (n_samples, n_params)
+ The sampling locations in the input space.
+
+ """
+ if not hasattr(self, 'raw_data'):
+ self.init_param_space(max_deg)
+ else:
+ if np.array(self.raw_data).ndim !=2:
+ raise AttributeError('The given raw data for sampling should have two dimensions')
+ samples = np.zeros((n_samples, self.ndim))
+ sample_size = self.raw_data.shape[1]
+
+ # Use a combination of raw data
+ if n_samples < sample_size:
+ for pa_idx in range(self.ndim):
+ # draw random indices
+ rand_idx = np.random.randint(0, sample_size, n_samples)
+ # store the raw data with given random indices
+ samples[:, pa_idx] = self.raw_data[pa_idx, rand_idx]
+ else:
+ try:
+ samples = self.JDist.resample(int(n_samples)).T
+ except AttributeError:
+ samples = self.JDist.sample(int(n_samples)).T
+ # Check if all samples are in the bound_tuples
+ for idx, param_set in enumerate(samples):
+ if not check_ranges(param_set, self.bound_tuples):
+ try:
+ proposed_sample = chaospy.generate_samples(
+ 1, domain=self.JDist, rule='random').T[0]
+ except:
+ proposed_sample = self.JDist.resample(1).T[0]
+ while not check_ranges(proposed_sample,
+ self.bound_tuples):
+ try:
+ proposed_sample = chaospy.generate_samples(
+ 1, domain=self.JDist, rule='random').T[0]
+ except:
+ proposed_sample = self.JDist.resample(1).T[0]
+ samples[idx] = proposed_sample
+
+ return samples
+
+ # -------------------------------------------------------------------------
+ def pcm_sampler(self, n_samples, max_deg):
+ """
+ Generates collocation points based on the root of the polynomial
+ degrees.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+ max_deg : int
+ Maximum degree defined by user. Will also be used to run
+ init_param_space if that has not been done beforehand.
+
+ Returns
+ -------
+ opt_col_points: array of shape (n_samples, n_params)
+ Collocation points.
+
+ """
+
+ if not hasattr(self, 'raw_data'):
+ self.init_param_space(max_deg)
+
+ raw_data = self.raw_data
+
+ # Guess the closest degree to self.n_samples
+ def M_uptoMax(deg):
+ result = []
+ for d in range(1, deg+1):
+ result.append(math.factorial(self.ndim+d) //
+ (math.factorial(self.ndim) * math.factorial(d)))
+ return np.array(result)
+ #print(M_uptoMax(max_deg))
+ #print(np.where(M_uptoMax(max_deg) > n_samples)[0])
+
+ guess_Deg = np.where(M_uptoMax(max_deg) > n_samples)[0][0]
+
+ c_points = np.zeros((guess_Deg+1, self.ndim))
+
+ def PolynomialPa(parIdx):
+ return apoly_construction(self.raw_data[parIdx], max_deg)
+
+ for i in range(self.ndim):
+ poly_coeffs = PolynomialPa(i)[guess_Deg+1][::-1]
+ c_points[:, i] = np.trim_zeros(np.roots(poly_coeffs))
+
+ # Construction of optimal integration points
+ Prod = itertools.product(np.arange(1, guess_Deg+2), repeat=self.ndim)
+ sort_dig_unique_combos = np.array(list(filter(lambda x: x, Prod)))
+
+ # Ranking relatively mean
+ Temp = np.empty(shape=[0, guess_Deg+1])
+ for j in range(self.ndim):
+ s = abs(c_points[:, j]-np.mean(raw_data[j]))
+ Temp = np.append(Temp, [s], axis=0)
+ temp = Temp.T
+
+ index_CP = np.sort(temp, axis=0)
+ sort_cpoints = np.empty((0, guess_Deg+1))
+
+ for j in range(self.ndim):
+ #print(index_CP[:, j])
+ sort_cp = c_points[index_CP[:, j], j]
+ sort_cpoints = np.vstack((sort_cpoints, sort_cp))
+
+ # Mapping of Combination to Cpoint Combination
+ sort_unique_combos = np.empty(shape=[0, self.ndim])
+ for i in range(len(sort_dig_unique_combos)):
+ sort_un_comb = []
+ for j in range(self.ndim):
+ SortUC = sort_cpoints[j, sort_dig_unique_combos[i, j]-1]
+ sort_un_comb.append(SortUC)
+ sort_uni_comb = np.asarray(sort_un_comb)
+ sort_unique_combos = np.vstack((sort_unique_combos, sort_uni_comb))
+
+ # Output the collocation points
+ if self.sampling_method.lower() == 'lscm':
+ opt_col_points = sort_unique_combos
+ else:
+ opt_col_points = sort_unique_combos[0:self.n_samples]
+
+ return opt_col_points
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/exploration.py b/examples/analytical-function/bayesvalidrox/surrogate_models/exploration.py
new file mode 100644
index 0000000000000000000000000000000000000000..6abb652f145fadb410ecf8f987142e8ceb544a41
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/exploration.py
@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Exploration for sequential training of metamodels
+"""
+
+import numpy as np
+from scipy.spatial import distance
+
+
+class Exploration:
+ """
+ Created based on the Surrogate Modeling Toolbox (SUMO) [1].
+
+ [1] Gorissen, D., Couckuyt, I., Demeester, P., Dhaene, T. and Crombecq, K.,
+ 2010. A surrogate modeling and adaptive sampling toolbox for computer
+ based design. Journal of machine learning research.-Cambridge, Mass.,
+ 11, pp.2051-2055. sumo@sumo.intec.ugent.be - http://sumo.intec.ugent.be
+
+ Attributes
+ ----------
+ ExpDesign : obj
+ ExpDesign object.
+ n_candidate : int
+ Number of candidate samples.
+ mc_criterion : str
+ Selection crieterion. The default is `'mc-intersite-proj-th'`. Another
+ option is `'mc-intersite-proj'`.
+ w : int
+ Number of random points in the domain for each sample of the
+ training set.
+ """
+
+ def __init__(self, ExpDesign, n_candidate,
+ mc_criterion='mc-intersite-proj-th'):
+ self.ExpDesign = ExpDesign
+ self.n_candidate = n_candidate
+ self.mc_criterion = mc_criterion
+ self.w = 100
+
+ def get_exploration_samples(self):
+ """
+ This function generates candidates to be selected as new design and
+ their associated exploration scores.
+
+ Returns
+ -------
+ all_candidates : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+ explore_method = self.ExpDesign.explore_method
+
+ print("\n")
+ print(f' The {explore_method}-Method is selected as the exploration '
+ 'method.')
+ print("\n")
+
+ if explore_method == 'Voronoi':
+ # Generate samples using the Voronoi method
+ all_candidates, exploration_scores = self.get_vornoi_samples()
+ else:
+ # Generate samples using the MC method
+ all_candidates, exploration_scores = self.get_mc_samples()
+
+ return all_candidates, exploration_scores
+
+ # -------------------------------------------------------------------------
+ def get_vornoi_samples(self):
+ """
+ This function generates samples based on voronoi cells and their
+ corresponding scores
+
+ Returns
+ -------
+ new_samples : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+
+ mc_criterion = self.mc_criterion
+ n_candidate = self.n_candidate
+ # Get the Old ExpDesign #samples
+ old_ED_X = self.ExpDesign.X
+ ndim = old_ED_X.shape[1]
+
+ # calculate error #averageErrors
+ error_voronoi, all_candidates = self.approximate_voronoi(
+ self.w, old_ED_X
+ )
+
+ # Pick the best candidate point in the voronoi cell
+ # for each best sample
+ selected_samples = np.empty((0, ndim))
+ bad_samples = []
+
+ for index in range(len(error_voronoi)):
+
+ # get candidate new samples from voronoi tesselation
+ candidates = self.closest_points[index]
+
+ # get total number of candidates
+ n_new_samples = candidates.shape[0]
+
+ # still no candidate samples around this one, skip it!
+ if n_new_samples == 0:
+ print('The following sample has been skipped because there '
+ 'were no candidate samples around it...')
+ print(old_ED_X[index])
+ bad_samples.append(index)
+ continue
+
+ # find candidate that is farthest away from any existing sample
+ max_min_distance = 0
+ best_candidate = 0
+ min_intersite_dist = np.zeros((n_new_samples))
+ min_projected_dist = np.zeros((n_new_samples))
+
+ for j in range(n_new_samples):
+
+ new_samples = np.vstack((old_ED_X, selected_samples))
+
+ # find min distorted distance from all other samples
+ euclidean_dist = self._build_dist_matrix_point(
+ new_samples, candidates[j], do_sqrt=True)
+ min_euclidean_dist = np.min(euclidean_dist)
+ min_intersite_dist[j] = min_euclidean_dist
+
+ # Check if this is the maximum minimum distance from all other
+ # samples
+ if min_euclidean_dist >= max_min_distance:
+ max_min_distance = min_euclidean_dist
+ best_candidate = j
+
+ # Projected distance
+ projected_dist = distance.cdist(
+ new_samples, [candidates[j]], 'chebyshev')
+ min_projected_dist[j] = np.min(projected_dist)
+
+ if mc_criterion == 'mc-intersite-proj':
+ weight_euclidean_dist = 0.5 * ((n_new_samples+1)**(1/ndim) - 1)
+ weight_projected_dist = 0.5 * (n_new_samples+1)
+ total_dist_scores = weight_euclidean_dist * min_intersite_dist
+ total_dist_scores += weight_projected_dist * min_projected_dist
+
+ elif mc_criterion == 'mc-intersite-proj-th':
+ alpha = 0.5 # chosen (tradeoff)
+ d_min = 2 * alpha / n_new_samples
+ if any(min_projected_dist < d_min):
+ candidates = np.delete(
+ candidates, [min_projected_dist < d_min], axis=0
+ )
+ total_dist_scores = np.delete(
+ min_intersite_dist, [min_projected_dist < d_min],
+ axis=0
+ )
+ else:
+ total_dist_scores = min_intersite_dist
+ else:
+ raise NameError(
+ 'The MC-Criterion you requested is not available.'
+ )
+
+ # Add the best candidate to the list of new samples
+ best_candidate = np.argsort(total_dist_scores)[::-1][:n_candidate]
+ selected_samples = np.vstack(
+ (selected_samples, candidates[best_candidate])
+ )
+
+ self.new_samples = selected_samples
+ self.exploration_scores = np.delete(error_voronoi, bad_samples, axis=0)
+
+ return self.new_samples, self.exploration_scores
+
+ # -------------------------------------------------------------------------
+ def get_mc_samples(self, all_candidates=None):
+ """
+ This function generates random samples based on Global Monte Carlo
+ methods and their corresponding scores, based on [1].
+
+ [1] Crombecq, K., Laermans, E. and Dhaene, T., 2011. Efficient
+ space-filling and non-collapsing sequential design strategies for
+ simulation-based modeling. European Journal of Operational Research
+ , 214(3), pp.683-696.
+ DOI: https://doi.org/10.1016/j.ejor.2011.05.032
+
+ Implemented methods to compute scores:
+ 1) mc-intersite-proj
+ 2) mc-intersite-proj-th
+
+ Arguments
+ ---------
+ all_candidates : array, optional
+ Samples to compute the scores for. The default is `None`. In this
+ case, samples will be generated by defined model input marginals.
+
+ Returns
+ -------
+ new_samples : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+ explore_method = self.ExpDesign.explore_method
+ mc_criterion = self.mc_criterion
+ if all_candidates is None:
+ n_candidate = self.n_candidate
+ else:
+ n_candidate = all_candidates.shape[0]
+
+ # Get the Old ExpDesign #samples
+ old_ED_X = self.ExpDesign.X
+ ndim = old_ED_X.shape[1]
+
+ # ----- Compute the number of random points -----
+ if all_candidates is None:
+ # Generate MC Samples
+ all_candidates = self.ExpDesign.generate_samples(
+ self.n_candidate, explore_method
+ )
+ self.all_candidates = all_candidates
+
+ # initialization
+ min_intersite_dist = np.zeros((n_candidate))
+ min_projected_dist = np.zeros((n_candidate))
+
+ for i, candidate in enumerate(all_candidates):
+
+ # find candidate that is farthest away from any existing sample
+ maxMinDistance = 0
+
+ # find min distorted distance from all other samples
+ euclidean_dist = self._build_dist_matrix_point(
+ old_ED_X, candidate, do_sqrt=True
+ )
+ min_euclidean_dist = np.min(euclidean_dist)
+ min_intersite_dist[i] = min_euclidean_dist
+
+ # Check if this is the maximum minimum distance from all other
+ # samples
+ if min_euclidean_dist >= maxMinDistance:
+ maxMinDistance = min_euclidean_dist
+
+ # Projected distance
+ projected_dist = self._build_dist_matrix_point(
+ old_ED_X, candidate, 'chebyshev'
+ )
+ min_projected_dist[i] = np.min(projected_dist)
+
+ if mc_criterion == 'mc-intersite-proj':
+ weight_euclidean_dist = ((n_candidate+1)**(1/ndim) - 1) * 0.5
+ weight_projected_dist = (n_candidate+1) * 0.5
+ total_dist_scores = weight_euclidean_dist * min_intersite_dist
+ total_dist_scores += weight_projected_dist * min_projected_dist
+
+ elif mc_criterion == 'mc-intersite-proj-th':
+ alpha = 0.5 # chosen (tradeoff)
+ d_min = 2 * alpha / n_candidate
+ if any(min_projected_dist < d_min):
+ all_candidates = np.delete(
+ all_candidates, [min_projected_dist < d_min], axis=0
+ )
+ total_dist_scores = np.delete(
+ min_intersite_dist, [min_projected_dist < d_min], axis=0
+ )
+ else:
+ total_dist_scores = min_intersite_dist
+ else:
+ raise NameError('The MC-Criterion you requested is not available.')
+
+ self.new_samples = all_candidates
+ self.exploration_scores = total_dist_scores
+ self.exploration_scores /= np.nansum(total_dist_scores)
+
+ return self.new_samples, self.exploration_scores
+
+ # -------------------------------------------------------------------------
+ def approximate_voronoi(self, w, samples):
+ """
+ An approximate (monte carlo) version of Matlab's voronoi command.
+
+ Arguments
+ ---------
+ samples : array
+ Old experimental design to be used as center points for voronoi
+ cells.
+
+ Returns
+ -------
+ areas : array
+ An approximation of the voronoi cells' areas.
+ all_candidates: list of arrays
+ A list of samples in each voronoi cell.
+ """
+ n_samples = samples.shape[0]
+ ndim = samples.shape[1]
+
+ # Compute the number of random points
+ n_points = w * samples.shape[0]
+ # Generate w random points in the domain for each sample
+ points = self.ExpDesign.generate_samples(n_points, 'random')
+ self.all_candidates = points
+
+ # Calculate the nearest sample to each point
+ self.areas = np.zeros((n_samples))
+ self.closest_points = [np.empty((0, ndim)) for i in range(n_samples)]
+
+ # Compute the minimum distance from all the samples of old_ED_X for
+ # each test point
+ for idx in range(n_points):
+ # calculate the minimum distance
+ distances = self._build_dist_matrix_point(
+ samples, points[idx], do_sqrt=True
+ )
+ closest_sample = np.argmin(distances)
+
+ # Add to the voronoi list of the closest sample
+ self.areas[closest_sample] = self.areas[closest_sample] + 1
+ prev_closest_points = self.closest_points[closest_sample]
+ self.closest_points[closest_sample] = np.vstack(
+ (prev_closest_points, points[idx])
+ )
+
+ # Divide by the amount of points to get the estimated volume of each
+ # voronoi cell
+ self.areas /= n_points
+
+ self.perc = np.max(self.areas * 100)
+
+ self.errors = self.areas
+
+ return self.areas, self.all_candidates
+
+ # -------------------------------------------------------------------------
+ def _build_dist_matrix_point(self, samples, point, method='euclidean',
+ do_sqrt=False):
+ """
+ Calculates the intersite distance of all points in samples from point.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params)
+ The old experimental design.
+ point : array
+ A candidate point.
+ method : str
+ Distance method.
+ do_sqrt : bool, optional
+ Whether to return distances or squared distances. The default is
+ `False`.
+
+ Returns
+ -------
+ distances : array
+ Distances.
+
+ """
+ distances = distance.cdist(samples, np.array([point]), method)
+
+ # do square root?
+ if do_sqrt:
+ return distances
+ else:
+ return distances**2
+
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/glexindex.py b/examples/analytical-function/bayesvalidrox/surrogate_models/glexindex.py
new file mode 100644
index 0000000000000000000000000000000000000000..90877331ec121750e7f81e32a4b69edbc9a110ba
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/glexindex.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Multi indices for monomial exponents.
+Credit: Jonathan Feinberg
+https://github.com/jonathf/numpoly/blob/master/numpoly/utils/glexindex.py
+"""
+
+import numpy
+import numpy.typing
+
+
+def glexindex(start, stop=None, dimensions=1, cross_truncation=1.,
+ graded=False, reverse=False):
+ """
+ Generate graded lexicographical multi-indices for the monomial exponents.
+ Args:
+ start (Union[int, numpy.ndarray]):
+ The lower order of the indices. If array of int, counts as lower
+ bound for each axis.
+ stop (Union[int, numpy.ndarray, None]):
+ The maximum shape included. If omitted: stop <- start; start <- 0
+ If int is provided, set as largest total order. If array of int,
+ set as upper bound for each axis.
+ dimensions (int):
+ The number of dimensions in the expansion.
+ cross_truncation (float, Tuple[float, float]):
+ Use hyperbolic cross truncation scheme to reduce the number of
+ terms in expansion. If two values are provided, first is low bound
+ truncation, while the latter upper bound. If only one value, upper
+ bound is assumed.
+ graded (bool):
+ Graded sorting, meaning the indices are always sorted by the index
+ sum. E.g. ``(2, 2, 2)`` has a sum of 6, and will therefore be
+ consider larger than both ``(3, 1, 1)`` and ``(1, 1, 3)``.
+ reverse (bool):
+ Reversed lexicographical sorting meaning that ``(1, 3)`` is
+ considered smaller than ``(3, 1)``, instead of the opposite.
+ Returns:
+ list:
+ Order list of indices.
+ Examples:
+ >>> numpoly.glexindex(4).tolist()
+ [[0], [1], [2], [3]]
+ >>> numpoly.glexindex(2, dimensions=2).tolist()
+ [[0, 0], [1, 0], [0, 1]]
+ >>> numpoly.glexindex(start=2, stop=3, dimensions=2).tolist()
+ [[2, 0], [1, 1], [0, 2]]
+ >>> numpoly.glexindex([1, 2, 3]).tolist()
+ [[0, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 2]]
+ >>> numpoly.glexindex([1, 2, 3], cross_truncation=numpy.inf).tolist()
+ [[0, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 1], [0, 0, 2], [0, 1, 2]]
+ """
+ if stop is None:
+ start, stop = 0, start
+ start = numpy.array(start, dtype=int).flatten()
+ stop = numpy.array(stop, dtype=int).flatten()
+ start, stop, _ = numpy.broadcast_arrays(start, stop, numpy.empty(dimensions))
+
+ cross_truncation = cross_truncation*numpy.ones(2)
+
+ # Moved here from _glexindex
+ bound = stop.max()
+ dimensions = len(start)
+ start = numpy.clip(start, a_min=0, a_max=None)
+ dtype = numpy.uint8 if bound < 256 else numpy.uint16
+ range_ = numpy.arange(bound, dtype=dtype)
+ indices = range_[:, numpy.newaxis]
+
+ for idx in range(dimensions-1):
+
+ # Truncate at each step to keep memory usage low
+ if idx:
+ indices = indices[cross_truncate(indices, bound-1, cross_truncation[1])]
+
+ # Repeats the current set of indices.
+ # e.g. [0,1,2] -> [0,1,2,0,1,2,...,0,1,2]
+ indices = numpy.tile(indices, (bound, 1))
+
+ # Stretches ranges over the new dimension.
+ # e.g. [0,1,2] -> [0,0,...,0,1,1,...,1,2,2,...,2]
+ front = range_.repeat(len(indices)//bound)[:, numpy.newaxis]
+
+ # Puts them two together.
+ indices = numpy.column_stack((front, indices))
+
+ # Complete the truncation scheme
+ if dimensions == 1:
+ indices = indices[(indices >= start) & (indices < bound)]
+ else:
+ lower = cross_truncate(indices, start-1, cross_truncation[0])
+ upper = cross_truncate(indices, stop-1, cross_truncation[1])
+ indices = indices[lower ^ upper]
+
+ indices = numpy.array(indices, dtype=int).reshape(-1, dimensions)
+ if indices.size:
+ # moved here from glexsort
+ keys = indices.T
+ keys_ = numpy.atleast_2d(keys)
+ if reverse:
+ keys_ = keys_[::-1]
+
+ indices_sort = numpy.array(numpy.lexsort(keys_))
+ if graded:
+ indices_sort = indices_sort[numpy.argsort(
+ numpy.sum(keys_[:, indices_sort], axis=0))].T
+
+ indices = indices[indices_sort]
+ return indices
+
+def cross_truncate(indices, bound, norm):
+ r"""
+ Truncate of indices using L_p norm.
+ .. math:
+ L_p(x) = \sum_i |x_i/b_i|^p ^{1/p} \leq 1
+ where :math:`b_i` are bounds that each :math:`x_i` should follow.
+ Args:
+ indices (Sequence[int]):
+ Indices to be truncated.
+ bound (int, Sequence[int]):
+ The bound function for witch the indices can not be larger than.
+ norm (float, Sequence[float]):
+ The `p` in the `L_p`-norm. Support includes both `L_0` and `L_inf`.
+ Returns:
+ Boolean indices to ``indices`` with True for each index where the
+ truncation criteria holds.
+ Examples:
+ >>> indices = numpy.array(numpy.mgrid[:10, :10]).reshape(2, -1).T
+ >>> indices[cross_truncate(indices, 2, norm=0)].T
+ array([[0, 0, 0, 1, 2],
+ [0, 1, 2, 0, 0]])
+ >>> indices[cross_truncate(indices, 2, norm=1)].T
+ array([[0, 0, 0, 1, 1, 2],
+ [0, 1, 2, 0, 1, 0]])
+ >>> indices[cross_truncate(indices, [0, 1], norm=1)].T
+ array([[0, 0],
+ [0, 1]])
+ """
+ assert norm >= 0, "negative L_p norm not allowed"
+ bound = numpy.asfarray(bound).flatten()*numpy.ones(indices.shape[1])
+
+ if numpy.any(bound < 0):
+ return numpy.zeros((len(indices),), dtype=bool)
+
+ if numpy.any(bound == 0):
+ out = numpy.all(indices[:, bound == 0] == 0, axis=-1)
+ if numpy.any(bound):
+ out &= cross_truncate(indices[:, bound != 0], bound[bound != 0], norm=norm)
+ return out
+
+ if norm == 0:
+ out = numpy.sum(indices > 0, axis=-1) <= 1
+ out[numpy.any(indices > bound, axis=-1)] = False
+ elif norm == numpy.inf:
+ out = numpy.max(indices/bound, axis=-1) <= 1
+ else:
+ out = numpy.sum((indices/bound)**norm, axis=-1)**(1./norm) <= 1
+
+ assert numpy.all(out[numpy.all(indices == 0, axis=-1)])
+
+ return out
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/input_space.py b/examples/analytical-function/bayesvalidrox/surrogate_models/input_space.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e010d66f2933ec243bad756d8f2c5454808d802
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/input_space.py
@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Input space built from set prior distributions
+"""
+
+import numpy as np
+import chaospy
+import scipy.stats as st
+
+
+class InputSpace:
+ """
+ This class generates the input space for the metamodel from the
+ distributions provided using the `Input` object.
+
+ Attributes
+ ----------
+ Input : obj
+ Input object containing the parameter marginals, i.e. name,
+ distribution type and distribution parameters or available raw data.
+ meta_Model_type : str
+ Type of the meta_Model_type.
+
+ """
+
+ def __init__(self, Input, meta_Model_type='pce'):
+ self.InputObj = Input
+ self.meta_Model_type = meta_Model_type
+
+ # Other
+ self.apce = None
+ self.ndim = None
+
+ # Init
+ self.check_valid_inputs()
+
+
+ def check_valid_inputs(self)-> None:
+ """
+ Check if the given InputObj is valid to use for further calculations:
+ Has some Marginals
+ Marginals have valid priors
+ All Marginals given as the same type (samples vs dist)
+
+ Returns
+ -------
+ None
+
+ """
+ Inputs = self.InputObj
+ self.ndim = len(Inputs.Marginals)
+
+ # Check if PCE or aPCE metamodel is selected.
+ # TODO: test also for 'pce'??
+ if self.meta_Model_type.lower() == 'apce':
+ self.apce = True
+ else:
+ self.apce = False
+
+ # check if marginals given
+ if not self.ndim >=1:
+ raise AssertionError('Cannot build distributions if no marginals are given')
+
+ # check that each marginal is valid
+ for marginals in Inputs.Marginals:
+ if len(marginals.input_data) == 0:
+ if marginals.dist_type == None:
+ raise AssertionError('Not all marginals were provided priors')
+ break
+ if np.array(marginals.input_data).shape[0] and (marginals.dist_type != None):
+ raise AssertionError('Both samples and distribution type are given. Please choose only one.')
+ break
+
+ # Check if input is given as dist or input_data.
+ self.input_data_given = -1
+ for marg in Inputs.Marginals:
+ #print(self.input_data_given)
+ size = np.array(marg.input_data).shape[0]
+ #print(f'Size: {size}')
+ if size and abs(self.input_data_given) !=1:
+ self.input_data_given = 2
+ break
+ if (not size) and self.input_data_given > 0:
+ self.input_data_given = 2
+ break
+ if not size:
+ self.input_data_given = 0
+ if size:
+ self.input_data_given = 1
+
+ if self.input_data_given == 2:
+ raise AssertionError('Distributions cannot be built as the priors have different types')
+
+
+ # Get the bounds if input_data are directly defined by user:
+ if self.input_data_given:
+ for i in range(self.ndim):
+ low_bound = np.min(Inputs.Marginals[i].input_data)
+ up_bound = np.max(Inputs.Marginals[i].input_data)
+ Inputs.Marginals[i].parameters = [low_bound, up_bound]
+
+
+
+ # -------------------------------------------------------------------------
+ def init_param_space(self, max_deg=None):
+ """
+ Initializes parameter space.
+
+ Parameters
+ ----------
+ max_deg : int, optional
+ Maximum degree. The default is `None`.
+
+ Creates
+ -------
+ raw_data : array of shape (n_params, n_samples)
+ Raw data.
+ bound_tuples : list of tuples
+ A list containing lower and upper bounds of parameters.
+
+ """
+ # Recheck all before running!
+ self.check_valid_inputs()
+
+ Inputs = self.InputObj
+ ndim = self.ndim
+ rosenblatt_flag = Inputs.Rosenblatt
+ mc_size = 50000
+
+ # Save parameter names
+ self.par_names = []
+ for parIdx in range(ndim):
+ self.par_names.append(Inputs.Marginals[parIdx].name)
+
+ # Create a multivariate probability distribution
+ # TODO: change this to make max_deg obligatory? at least in some specific cases?
+ if max_deg is not None:
+ JDist, poly_types = self.build_polytypes(rosenblatt=rosenblatt_flag)
+ self.JDist, self.poly_types = JDist, poly_types
+
+ if self.input_data_given:
+ self.MCSize = len(Inputs.Marginals[0].input_data)
+ self.raw_data = np.zeros((ndim, self.MCSize))
+
+ for parIdx in range(ndim):
+ # Save parameter names
+ try:
+ self.raw_data[parIdx] = np.array(
+ Inputs.Marginals[parIdx].input_data)
+ except:
+ self.raw_data[parIdx] = self.JDist[parIdx].sample(mc_size)
+
+ else:
+ # Generate random samples based on parameter distributions
+ self.raw_data = chaospy.generate_samples(mc_size,
+ domain=self.JDist)
+
+ # Extract moments
+ for parIdx in range(ndim):
+ mu = np.mean(self.raw_data[parIdx])
+ std = np.std(self.raw_data[parIdx])
+ self.InputObj.Marginals[parIdx].moments = [mu, std]
+
+ # Generate the bounds based on given inputs for marginals
+ bound_tuples = []
+ for i in range(ndim):
+ if Inputs.Marginals[i].dist_type == 'unif':
+ low_bound = Inputs.Marginals[i].parameters[0]
+ up_bound = Inputs.Marginals[i].parameters[1]
+ else:
+ low_bound = np.min(self.raw_data[i])
+ up_bound = np.max(self.raw_data[i])
+
+ bound_tuples.append((low_bound, up_bound))
+
+ self.bound_tuples = tuple(bound_tuples)
+
+ # -------------------------------------------------------------------------
+ def build_polytypes(self, rosenblatt):
+ """
+ Creates the polynomial types to be passed to univ_basis_vals method of
+ the MetaModel object.
+
+ Parameters
+ ----------
+ rosenblatt : bool
+ Rosenblatt transformation flag.
+
+ Returns
+ -------
+ orig_space_dist : object
+ A chaospy JDist object or a gaussian_kde object.
+ poly_types : list
+ List of polynomial types for the parameters.
+
+ """
+ Inputs = self.InputObj
+
+ all_data = []
+ all_dist_types = []
+ orig_joints = []
+ poly_types = []
+
+ for parIdx in range(self.ndim):
+
+ if Inputs.Marginals[parIdx].dist_type is None:
+ data = Inputs.Marginals[parIdx].input_data
+ all_data.append(data)
+ dist_type = None
+ else:
+ dist_type = Inputs.Marginals[parIdx].dist_type
+ params = Inputs.Marginals[parIdx].parameters
+
+ if rosenblatt:
+ polytype = 'hermite'
+ dist = chaospy.Normal()
+
+ elif dist_type is None:
+ polytype = 'arbitrary'
+ dist = None
+
+ elif 'unif' in dist_type.lower():
+ polytype = 'legendre'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Uniform(lower=params[0], upper=params[1])
+
+ elif 'norm' in dist_type.lower() and \
+ 'log' not in dist_type.lower():
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ polytype = 'hermite'
+ dist = chaospy.Normal(mu=params[0], sigma=params[1])
+
+ elif 'gamma' in dist_type.lower():
+ polytype = 'laguerre'
+ if not np.array(params).shape[0]>=3:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Gamma(shape=params[0],
+ scale=params[1],
+ shift=params[2])
+
+ elif 'beta' in dist_type.lower():
+ if not np.array(params).shape[0]>=4:
+ raise AssertionError('Distribution has too few parameters!')
+ polytype = 'jacobi'
+ dist = chaospy.Beta(alpha=params[0], beta=params[1],
+ lower=params[2], upper=params[3])
+
+ elif 'lognorm' in dist_type.lower():
+ polytype = 'hermite'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ mu = np.log(params[0]**2/np.sqrt(params[0]**2 + params[1]**2))
+ sigma = np.sqrt(np.log(1 + params[1]**2 / params[0]**2))
+ dist = chaospy.LogNormal(mu, sigma)
+ # dist = chaospy.LogNormal(mu=params[0], sigma=params[1])
+
+ elif 'expon' in dist_type.lower():
+ polytype = 'exponential'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Exponential(scale=params[0], shift=params[1])
+
+ elif 'weibull' in dist_type.lower():
+ polytype = 'weibull'
+ if not np.array(params).shape[0]>=3:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Weibull(shape=params[0], scale=params[1],
+ shift=params[2])
+
+ else:
+ message = (f"DistType {dist_type} for parameter"
+ f"{parIdx+1} is not available.")
+ raise ValueError(message)
+
+ if self.input_data_given or self.apce:
+ polytype = 'arbitrary'
+
+ # Store dists and poly_types
+ orig_joints.append(dist)
+ poly_types.append(polytype)
+ all_dist_types.append(dist_type)
+
+ # Prepare final output to return
+ if None in all_dist_types:
+ # Naive approach: Fit a gaussian kernel to the provided data
+ Data = np.asarray(all_data)
+ try:
+ orig_space_dist = st.gaussian_kde(Data)
+ except:
+ raise ValueError('The samples provided to the Marginals should be 1D only')
+ self.prior_space = orig_space_dist
+ else:
+ orig_space_dist = chaospy.J(*orig_joints)
+ try:
+ self.prior_space = st.gaussian_kde(orig_space_dist.sample(10000))
+ except:
+ raise ValueError('Parameter values are not valid, please set differently')
+
+ return orig_space_dist, poly_types
+
+ # -------------------------------------------------------------------------
+ def transform(self, X, params=None, method=None):
+ """
+ Transforms the samples via either a Rosenblatt or an isoprobabilistic
+ transformation.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples,n_params)
+ Samples to be transformed.
+ method : string
+ If transformation method is 'user' transform X, else just pass X.
+
+ Returns
+ -------
+ tr_X: array of shape (n_samples,n_params)
+ Transformed samples.
+
+ """
+ # Check for built JDist
+ if not hasattr(self, 'JDist'):
+ raise AttributeError('Call function init_param_space first to create JDist')
+
+ # Check if X is 2d
+ if X.ndim != 2:
+ raise AttributeError('X should have two dimensions')
+
+ # Check if size of X matches Marginals
+ if X.shape[1]!= self.ndim:
+ raise AttributeError('The second dimension of X should be the same size as the number of marginals in the InputObj')
+
+ if self.InputObj.Rosenblatt:
+ self.origJDist, _ = self.build_polytypes(False)
+ if method == 'user':
+ tr_X = self.JDist.inv(self.origJDist.fwd(X.T)).T
+ else:
+ # Inverse to original spcace -- generate sample ED
+ tr_X = self.origJDist.inv(self.JDist.fwd(X.T)).T
+ else:
+ # Transform samples via an isoprobabilistic transformation
+ n_samples, n_params = X.shape
+ Inputs = self.InputObj
+ origJDist = self.JDist
+ poly_types = self.poly_types
+
+ disttypes = []
+ for par_i in range(n_params):
+ disttypes.append(Inputs.Marginals[par_i].dist_type)
+
+ # Pass non-transformed X, if arbitrary PCE is selected.
+ if None in disttypes or self.input_data_given or self.apce:
+ return X
+
+ cdfx = np.zeros((X.shape))
+ tr_X = np.zeros((X.shape))
+
+ for par_i in range(n_params):
+
+ # Extract the parameters of the original space
+ disttype = disttypes[par_i]
+ if disttype is not None:
+ dist = origJDist[par_i]
+ else:
+ dist = None
+ polytype = poly_types[par_i]
+ cdf = np.vectorize(lambda x: dist.cdf(x))
+
+ # Extract the parameters of the transformation space based on
+ # polyType
+ if polytype == 'legendre' or disttype == 'uniform':
+ # Generate Y_Dists based
+ params_Y = [-1, 1]
+ dist_Y = st.uniform(loc=params_Y[0],
+ scale=params_Y[1]-params_Y[0])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ elif polytype == 'hermite' or disttype == 'norm':
+ params_Y = [0, 1]
+ dist_Y = st.norm(loc=params_Y[0], scale=params_Y[1])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ elif polytype == 'laguerre' or disttype == 'gamma':
+ if params == None:
+ raise AttributeError('Additional parameters have to be set for the gamma distribution!')
+ params_Y = [1, params[1]]
+ dist_Y = st.gamma(loc=params_Y[0], scale=params_Y[1])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ # Compute CDF_x(X)
+ cdfx[:, par_i] = cdf(X[:, par_i])
+
+ # Compute invCDF_y(cdfx)
+ tr_X[:, par_i] = inv_cdf(cdfx[:, par_i])
+
+ return tr_X
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/inputs.py b/examples/analytical-function/bayesvalidrox/surrogate_models/inputs.py
new file mode 100644
index 0000000000000000000000000000000000000000..094e1066fe008e37288e44750524c5a1370bd7a2
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/inputs.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Inputs and related marginal distributions
+"""
+
+class Input:
+ """
+ A class to define the uncertain input parameters.
+
+ Attributes
+ ----------
+ Marginals : obj
+ Marginal objects. See `inputs.Marginal`.
+ Rosenblatt : bool
+ If Rossenblatt transformation is required for the dependent input
+ parameters.
+
+ Examples
+ -------
+ Marginals can be defined as following:
+
+ >>> Inputs.add_marginals()
+ >>> Inputs.Marginals[0].name = 'X_1'
+ >>> Inputs.Marginals[0].dist_type = 'uniform'
+ >>> Inputs.Marginals[0].parameters = [-5, 5]
+
+ If there is no common data is avaliable, the input data can be given
+ as following:
+
+ >>> Inputs.add_marginals()
+ >>> Inputs.Marginals[0].name = 'X_1'
+ >>> Inputs.Marginals[0].input_data = input_data
+ """
+ poly_coeffs_flag = True
+
+ def __init__(self):
+ self.Marginals = []
+ self.Rosenblatt = False
+
+ def add_marginals(self):
+ """
+ Adds a new Marginal object to the input object.
+
+ Returns
+ -------
+ None.
+
+ """
+ self.Marginals.append(Marginal())
+
+
+# Nested class
+class Marginal:
+ """
+ An object containing the specifications of the marginals for each uncertain
+ parameter.
+
+ Attributes
+ ----------
+ name : string
+ Name of the parameter. The default is `'$x_1$'`.
+ dist_type : string
+ Name of the distribution. The default is `None`.
+ parameters : list
+ List of the parameters corresponding to the distribution type. The
+ default is `None`.
+ input_data : array
+ Available input data. The default is `[]`.
+ moments : list
+ List of the moments.
+ """
+
+ def __init__(self):
+ self.name = '$x_1$'
+ self.dist_type = None
+ self.parameters = None
+ self.input_data = []
+ self.moments = None
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/meta_model_engine.py b/examples/analytical-function/bayesvalidrox/surrogate_models/meta_model_engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..71c0244216b0c87a22174a3ad2043a4c0a80efab
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/meta_model_engine.py
@@ -0,0 +1,2195 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 28 09:21:18 2022
+
+@author: farid
+"""
+import numpy as np
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from copy import deepcopy, copy
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import sys
+import os
+import gc
+import seaborn as sns
+from joblib import Parallel, delayed
+
+import bayesvalidrox
+from .exploration import Exploration
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+from bayesvalidrox.bayes_inference.discrepancy import Discrepancy
+import pandas as pd
+
+
+class MetaModelEngine():
+ """ Sequential experimental design
+ This class provieds method for trainig the meta-model in an iterative
+ manners.
+ The main method to execute the task is `train_seq_design`, which
+ recieves a model object and returns the trained metamodel.
+ """
+
+ def __init__(self, meta_model_opts):
+ self.MetaModel = meta_model_opts
+
+ # -------------------------------------------------------------------------
+ def run(self):
+
+ Model = self.MetaModel.ModelObj
+ self.MetaModel.n_params = len(self.MetaModel.input_obj.Marginals)
+ self.MetaModel.ExpDesignFlag = 'normal'
+ # --- Prepare pce degree ---
+ if self.MetaModel.meta_model_type.lower() == 'pce':
+ if type(self.MetaModel.pce_deg) is not np.ndarray:
+ self.MetaModel.pce_deg = np.array(self.MetaModel.pce_deg)
+
+ if self.MetaModel.ExpDesign.method == 'normal':
+ self.MetaModel.ExpDesignFlag = 'normal'
+ self.MetaModel.train_norm_design(parallel = False)
+
+ elif self.MetaModel.ExpDesign.method == 'sequential':
+ self.train_seq_design()
+ else:
+ raise Exception("The method for experimental design you requested"
+ " has not been implemented yet.")
+
+ # Zip the model run directories
+ if self.MetaModel.ModelObj.link_type.lower() == 'pylink' and\
+ self.MetaModel.ExpDesign.sampling_method.lower() != 'user':
+ Model.zip_subdirs(Model.name, f'{Model.name}_')
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ # Set model to have shorter call
+ Model = self.MetaModel.ModelObj
+ # MetaModel = self.MetaModel
+ self.Model = Model
+
+ # Initialization
+ self.MetaModel.SeqModifiedLOO = {}
+ self.MetaModel.seqValidError = {}
+ self.MetaModel.SeqBME = {}
+ self.MetaModel.SeqKLD = {}
+ self.MetaModel.SeqDistHellinger = {}
+ self.MetaModel.seqRMSEMean = {}
+ self.MetaModel.seqRMSEStd = {}
+ self.MetaModel.seqMinDist = []
+
+ # Determine the metamodel type
+ if self.MetaModel.meta_model_type.lower() != 'gpe':
+ pce = True
+ else:
+ pce = False
+ # If given, use mc reference data
+ mc_ref = True if bool(Model.mc_reference) else False
+ if mc_ref:
+ Model.read_mc_reference()
+
+ # if valid_samples not defined, do so now
+ if not hasattr(self.MetaModel, 'valid_samples'):
+ self.MetaModel.valid_samples = []
+ self.MetaModel.valid_model_runs = []
+ self.MetaModel.valid_likelihoods = []
+
+ # Get the parameters
+ max_n_samples = self.MetaModel.ExpDesign.n_max_samples
+ mod_LOO_threshold = self.MetaModel.ExpDesign.mod_LOO_threshold
+ n_canddidate = self.MetaModel.ExpDesign.n_canddidate
+ post_snapshot = self.MetaModel.ExpDesign.post_snapshot
+ n_replication = self.MetaModel.ExpDesign.n_replication
+ util_func = self.MetaModel.ExpDesign.util_func
+ output_name = Model.Output.names
+ validError = None
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [self.MetaModel.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ if len(Model.observations) != 0 or Model.meas_file is not None:
+ self.observations = Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ TotalSigma2 = {}
+
+ # TODO: ---------- Initial self.MetaModel ----------
+ # First run MetaModel on non-sequential design
+ self.MetaModel.train_norm_design(parallel = False)
+ initMetaModel = deepcopy(self.MetaModel)
+
+ # Validation error if validation set is provided. - use as initial errors
+ if self.MetaModel.valid_model_runs:
+ init_rmse, init_valid_error = self.__validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ TotalSigma2 = self.MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self.__BME_Calculator(
+ initMetaModel, obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = self.MetaModel.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self.__error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean:.2f},"
+ f" {init_rmse_std:.2f}")
+
+ # Read the initial experimental design
+ # TODO: this sequential, or the non-sequential samples??
+ Xinit = initMetaModel.ExpDesign.X
+ init_n_samples = len(initMetaModel.ExpDesign.X)
+ initYprev = initMetaModel.ModelOutputDict
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ # Replicate the sequential design
+ for repIdx in range(n_replication): # TODO: what does this do?
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # To avoid changes ub original aPCE object
+ self.MetaModel.ExpDesign.X = Xinit
+ self.MetaModel.ExpDesign.Y = initYprev
+ self.MetaModel.LCerror = initLCerror
+
+ for util_f in util_func: # TODO: recheck choices for this
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ self.MetaModel.ExpDesign.X = Xinit
+ self.MetaModel.ExpDesign.Y = initYprev
+ self.MetaModel.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(self.MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(self.MetaModel) # Write last MetaModel here
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel( # What's the use of this here??
+ samples=Xfull[-1].reshape(1, -1))
+ del prevMetaModel_dict[itr_no-1] # Delete second to last metamodel here?
+
+ # Optimal Bayesian Design
+ self.MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.opt_SeqDesign(TotalSigma2, # TODO: check in this!!
+ n_canddidate,
+ util_f)
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ self.MetaModel.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(self.MetaModel, 'adapt_verbose') and \
+ self.MetaModel.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = self.MetaModel.eval_metamodel(
+ samples=Xnew
+ )
+ adaptPlot(
+ self.MetaModel, Ynew, y_hat, std_hat,
+ plotED=False
+ )
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ self.MetaModel.ModelOutputDict[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ self.MetaModel.ExpDesign.sampling_method = 'user'
+ self.MetaModel.ExpDesign.X = Xfull
+ self.MetaModel.ExpDesign.Y = self.MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = self.MetaModel.ModelOutputDict
+
+ # Pass the new prior as the input
+ self.MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ self.MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ self.MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ self.MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ self.MetaModel.train_norm_design(parallel=False)
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if self.MetaModel.valid_model_runs:
+ rmse, validError = self.__validError(self.MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(self.MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self.__BME_Calculator(self.MetaModel, obs_data,
+ TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = self.MetaModel.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = self.MetaModel.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self.__error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ self.MetaModel.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(self.MetaModel.valid_model_runs) != 0:
+ self.MetaModel.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ self.MetaModel.SeqBME[strKey] = SeqBME
+ self.MetaModel.SeqKLD[strKey] = SeqKLD
+ if hasattr(self.MetaModel, 'valid_likelihoods') and \
+ self.MetaModel.valid_likelihoods:
+ self.MetaModel.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ self.MetaModel.seqRMSEMean[strKey] = seqRMSEMean
+ self.MetaModel.seqRMSEStd[strKey] = seqRMSEStd
+
+ # return self.MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = MetaModel.ExpDesign.X
+ out_dict_y = MetaModel.ExpDesign.Y
+ out_names = MetaModel.ModelObj.Output.names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, y_hat, std, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Get the data
+ obs_data = self.observations
+ n_obs = self.Model.n_obs
+ mc_size = 10000
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((mc_size))
+ for key in list(y_hat):
+ cov = np.diag(std[key]**2)
+ rv = stats.multivariate_normal(mean=y_hat[key], cov=cov)
+ Y_MC[key] = rv.rvs(size=mc_size)
+ logPriorLikelihoods += rv.logpdf(Y_MC[key])
+ std_MC[key] = np.zeros((mc_size, y_hat[key].shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, mc_size)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods), dtype=np.longdouble)
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = np.nanmean(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self.__normpdf(
+ y_hat, std, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def update_metamodel(self, MetaModel, output, y_hat_can, univ_p_val, index,
+ new_pca=False):
+ BasisIndices = MetaModel.basis_dict[output]["y_"+str(index+1)]
+ clf_poly = MetaModel.clf_poly[output]["y_"+str(index+1)]
+ Mn = clf_poly.coef_
+ Sn = clf_poly.sigma_
+ beta = clf_poly.alpha_
+ active = clf_poly.active_
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ Sn_new_inv = np.linalg.inv(Sn)
+ Sn_new_inv += beta * np.dot(Psi[:, active].T, Psi[:, active])
+ Sn_new = np.linalg.inv(Sn_new_inv)
+
+ Mn_new = np.dot(Sn_new_inv, Mn[active]).reshape(-1, 1)
+ Mn_new += beta * np.dot(Psi[:, active].T, y_hat_can)
+ Mn_new = np.dot(Sn_new, Mn_new).flatten()
+
+ # Compute the old and new moments of PCEs
+ mean_old = Mn[0]
+ mean_new = Mn_new[0]
+ std_old = np.sqrt(np.sum(np.square(Mn[1:])))
+ std_new = np.sqrt(np.sum(np.square(Mn_new[1:])))
+
+ # Back transformation if PCA is selected.
+ if MetaModel.dim_red_method.lower() == 'pca':
+ old_pca = MetaModel.pca[output]
+ mean_old = old_pca.mean_[index]
+ mean_old += np.sum(mean_old * old_pca.components_[:, index])
+ std_old = np.sqrt(np.sum(std_old**2 *
+ old_pca.components_[:, index]**2))
+ mean_new = new_pca.mean_[index]
+ mean_new += np.sum(mean_new * new_pca.components_[:, index])
+ std_new = np.sqrt(np.sum(std_new**2 *
+ new_pca.components_[:, index]**2))
+ # print(f"mean_old: {mean_old:.2f} mean_new: {mean_new:.2f}")
+ # print(f"std_old: {std_old:.2f} std_new: {std_new:.2f}")
+ # Store the old and new moments of PCEs
+ results = {
+ 'mean_old': mean_old,
+ 'mean_new': mean_new,
+ 'std_old': std_old,
+ 'std_new': std_new
+ }
+ return results
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign_old(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ old_EDY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels using the candidate design
+ Y_PC_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Generate y from posterior predictive
+ m_size = 100
+ y_hat_samples = {}
+ for idx, key in enumerate(Model.Output.names):
+ means, stds = Y_PC_can[key][0], Y_std_can[key][0]
+ y_hat_samples[key] = np.random.multivariate_normal(
+ means, np.diag(stds), m_size)
+
+ # Create the SparseBayes-based PCE metamodel:
+ MetaModel.input_obj.poly_coeffs_flag = False
+ univ_p_val = self.MetaModel.univ_basis_vals(X_can)
+ G_n_m_all = np.zeros((m_size, len(Model.Output.names), Model.n_obs))
+
+ for i in range(m_size):
+ for idx, key in enumerate(Model.Output.names):
+ if MetaModel.dim_red_method.lower() == 'pca':
+ # Equal number of components
+ new_outputs = np.vstack(
+ (old_EDY[key], y_hat_samples[key][i])
+ )
+ new_pca, _ = MetaModel.pca_transformation(new_outputs)
+ target = new_pca.transform(
+ y_hat_samples[key][i].reshape(1, -1)
+ )[0]
+ else:
+ new_pca, target = False, y_hat_samples[key][i]
+
+ for j in range(len(target)):
+
+ # Update surrogate
+ result = self.update_metamodel(
+ MetaModel, key, target[j], univ_p_val, j, new_pca)
+
+ # Compute Expected Information Gain (Eq. 39)
+ G_n_m = np.log(result['std_old']/result['std_new']) - 1./2
+ G_n_m += result['std_new']**2 / (2*result['std_old']**2)
+ G_n_m += (result['mean_new'] - result['mean_old'])**2 /\
+ (2*result['std_old']**2)
+
+ G_n_m_all[i, idx, j] = G_n_m
+
+ U_J_d = G_n_m_all.mean(axis=(1, 2)).mean()
+ return -1 * U_J_d
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ MetaModel = self.MetaModel
+ out_names = MetaModel.ModelObj.Output.names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+ oldExpDesignY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+ PCE_Model_can = deepcopy(MetaModel)
+ # Add the candidate to the ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ NewExpDesignY[key] = np.vstack(
+ (oldExpDesignY[key], Y_PC_can[key])
+ )
+
+ PCE_Model_can.ExpDesign.sampling_method = 'user'
+ PCE_Model_can.ExpDesign.X = NewExpDesignX
+ PCE_Model_can.ModelOutputDict = NewExpDesignY
+ PCE_Model_can.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ PCE_Model_can.input_obj.poly_coeffs_flag = False
+ PCE_Model_can.train_norm_design(parallel=False)
+
+ # Set the ExpDesign to its original values
+ PCE_Model_can.ExpDesign.X = oldExpDesignX
+ PCE_Model_can.ModelOutputDict = oldExpDesignY
+ PCE_Model_can.ExpDesign.Y = oldExpDesignY
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = PCE_Model_can.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.MetaModel.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.longdouble))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postLikelihoods /= np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def subdomain(self, Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ Subdomains : TYPE
+ DESCRIPTION.
+
+ """
+ n_params = self.MetaModel.n_params
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ # Evaluate all candidates
+ y_can, std_can = self.MetaModel.eval_metamodel(samples=candidates)
+ # loop through candidates
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="BAL Design"):
+ y_hat = {key: items[idx] for key, items in y_can.items()}
+ std = {key: items[idx] for key, items in std_can.items()}
+ U_J_d[idx] = self.util_BayesianActiveDesign(
+ y_hat, std, sigma2Dict, var)
+
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithim to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.MetaModel.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def opt_SeqDesign(self, sigma2, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None.
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ MetaModel = self.MetaModel
+ Bounds = MetaModel.bound_tuples
+ n_new_samples = MetaModel.ExpDesign.n_new_samples
+ explore_method = MetaModel.ExpDesign.explore_method
+ exploit_method = MetaModel.ExpDesign.exploit_method
+ n_cand_groups = MetaModel.ExpDesign.n_cand_groups
+ tradeoff_scheme = MetaModel.ExpDesign.tradeoff_scheme
+
+ old_EDX = MetaModel.ExpDesign.X
+ old_EDY = MetaModel.ExpDesign.Y.copy()
+ ndim = MetaModel.ExpDesign.X.shape[1]
+ OutputNames = MetaModel.ModelObj.Output.names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+
+ Xnew, filteredSamples = MetaModel.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ args = []
+ subdomains = self.subdomain(Bounds, n_new_samples)
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+
+ print("\nXnew:\n", Xnew)
+
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ elif explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = MetaModel.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(MetaModel, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = MetaModel.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = MetaModel.ExpDesign.generate_samples(
+ MetaModel.ExpDesign.max_func_itr, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+
+ results = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ # out = map(self.run_util_func,
+ # [exploit_method]*n_cand_groups,
+ # split_cand,
+ # range(n_cand_groups),
+ # [sigma2] * n_cand_groups,
+ # [var] * n_cand_groups,
+ # [X_MC] * n_cand_groups
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # create surrogate model for U_J_d
+ # from sklearn.preprocessing import MinMaxScaler
+ # # Take care of inf entries
+ # good_indices = [i for i, arr in enumerate(U_J_d)
+ # if np.isfinite(arr).all()]
+ # scaler = MinMaxScaler()
+ # X_S = scaler.fit_transform(candidates[good_indices])
+ # gp = MetaModel.gaussian_process_emulator(
+ # X_S, U_J_d[good_indices], autoSelect=False
+ # )
+ # U_J_d = gp.predict(scaler.transform(allCandidates))
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Accumulate the samples
+ # TODO: added this, recheck!!
+ finalCandidates = np.concatenate((allCandidates, candidates), axis = 0)
+ finalCandidates = np.unique(finalCandidates, axis = 0)
+
+ #self.allCandidates = allCandidates
+ #self.candidates = candidates
+ #self.norm_U_J_d = norm_U_J_d
+ #self.exploit_w = exploit_w
+ #self.scoreExploration = scoreExploration
+
+ # Total score
+ #totalScore = exploit_w * norm_U_J_d
+ #totalScore += explore_w * scoreExploration
+
+ # TODO: changed this from the above to take into account both exploration and exploitation samples without duplicates
+ totalScore = np.zeros(finalCandidates.shape[0])
+ #self.totalScore = totalScore
+
+ for cand_idx in range(finalCandidates.shape[0]):
+ # find candidate indices
+ idx1 = np.where(allCandidates == finalCandidates[cand_idx])[0]
+ idx2 = np.where(candidates == finalCandidates[cand_idx])[0]
+ #print(f'Candidate number {cand_idx}')
+ #print(finalCandidates[cand_idx])
+ #print(f'Idx1: {idx1}, Idx2: {idx2}')
+
+ # exploration
+ if idx1 != []:
+ idx1 = idx1[0]
+ #print(f'Values1: {allCandidates[idx1]}')
+ totalScore[cand_idx] += explore_w * scoreExploration[idx1]
+
+ # exploitation
+ if idx2 != []:
+ idx2 = idx2[0]
+ #print(f'Values1: {candidates[idx2]}')
+ totalScore[cand_idx] += exploit_w * norm_U_J_d[idx2]
+
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ # TODO: changed this from allCandiates to full set of candidates - still not changed for e.g. 'Voronoi'
+ Xnew = finalCandidates[sorted_idxtotalScore[:n_new_samples]] # here candidates(exploitation) vs allCandidates (exploration)!!
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+ # out = map(self.run_util_func,
+ # [exploit_method]*len(goodSampleIdx),
+ # split_cand,
+ # range(len(goodSampleIdx)),
+ # [sigma2] * len(goodSampleIdx),
+ # [var] * len(goodSampleIdx)
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self
+ Model = self.Model
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = Model.Output.names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ Scores = list(MetaModel.score_dict[OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ BasisIndices = MetaModelOrig.basis_dict[OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = MetaModelOrig.univ_basis_vals(oldExpDesignX)
+ Psi = MetaModelOrig.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.univ_basis_vals(candidates)
+ Psi_c = self.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var == 'D-Opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var == 'A-Opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var == 'K-Opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def __normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+
+ Model = self.Model
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(Model.Output.names):
+
+ # (Meta)Model Output
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+ else:
+ tot_sigma2s += np.mean(std_pce[out])**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = MetaModel.ExpDesign.X # valid_samples
+ model_outputs = MetaModel.ExpDesign.Y # valid_model_runs
+ Model = MetaModel.ModelObj
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self.__logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self.__logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def __logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def __posteriorPlot(self, posterior, par_names, key):
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.MetaModel.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+ # -------------------------------------------------------------------------
+ def __hellinger_distance(self, P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+ # -------------------------------------------------------------------------
+ def __BME_Calculator(self, MetaModel, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ """
+ # Initializations
+ if hasattr(MetaModel, 'valid_likelihoods'):
+ valid_likelihoods = MetaModel.valid_likelihoods
+ else:
+ valid_likelihoods = []
+
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ #print(f'post_snapshot: {post_snapshot}')
+ if post_snapshot or len(valid_likelihoods) != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = MetaModel.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ Y_MC, std_MC = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self.__normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(MetaModel.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or valid_likelihoods:
+ # Hellinger distance
+ #print('arrived here')
+ #print(np.array(valid_likelihoods))
+ valid_likelihoods = np.array(valid_likelihoods)
+ #valid_likelihoods = np.array(valid_likelihoods)
+ ref_like = np.log(valid_likelihoods[(valid_likelihoods > 0)])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = self.__hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and MetaModel.n_params == 2 and not idx % 5:
+ BayesOpts = BayesInference(MetaModel)
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ obs_data = pd.DataFrame(obs_data, columns=self.Model.Output.names)
+ BayesOpts.measurement_error = obs_data
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def __validError(self, MetaModel):
+
+ # MetaModel = self.MetaModel
+ Model = MetaModel.ModelObj
+ OutputName = Model.Output.names
+
+ # Extract the original model with the generated samples
+ valid_samples = MetaModel.valid_samples
+ valid_model_runs = MetaModel.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ valid_PCE_runs, _ = MetaModel.eval_metamodel(samples=valid_samples)
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in OutputName:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def __error_Mean_Std(self):
+
+ MetaModel = self.MetaModel
+ # Extract the mean and std provided by user
+ df_MCReference = MetaModel.ModelObj.mc_reference
+
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+
+ # Compute the root mean squared error
+ for output in MetaModel.ModelObj.Output.names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ df_MCReference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ df_MCReference['std'], pce_means[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self, MetaModel):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+ outputs = MetaModel.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = PCA.inverse_transform(np.sqrt(pce_var))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py b/examples/analytical-function/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py
new file mode 100644
index 0000000000000000000000000000000000000000..96ef9c1d50b10b587ad0846d41733fc7f1cedfe8
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py
@@ -0,0 +1,366 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jul 15 14:08:59 2022
+
+@author: farid
+"""
+import numpy as np
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+from sklearn.utils import check_X_y
+
+
+def corr(x, y):
+ return abs(x.dot(y))/np.sqrt((x**2).sum())
+
+
+class OrthogonalMatchingPursuit(LinearModel, RegressorMixin):
+ '''
+ Regression with Orthogonal Matching Pursuit [1].
+
+ Parameters
+ ----------
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ References
+ ----------
+ [1] Pati, Y., Rezaiifar, R., Krishnaprasad, P. (1993). Orthogonal matching
+ pursuit: recursive function approximation with application to wavelet
+ decomposition. Proceedings of 27th Asilomar Conference on Signals,
+ Systems and Computers, 40-44.
+ '''
+
+ def __init__(self, fit_intercept=True, normalize=False, copy_X=True,
+ verbose=False):
+ self.fit_intercept = fit_intercept
+ self.normalize = normalize
+ self.copy_X = copy_X
+ self.verbose = verbose
+
+ def _preprocess_data(self, X, y):
+ """Center and scale data.
+ Centers data to have mean zero along axis 0. If fit_intercept=False or
+ if the X is a sparse matrix, no centering is done, but normalization
+ can still be applied. The function returns the statistics necessary to
+ reconstruct the input data, which are X_offset, y_offset, X_scale, such
+ that the output
+ X = (X - X_offset) / X_scale
+ X_scale is the L2 norm of X - X_offset.
+ """
+
+ if self.copy_X:
+ X = X.copy(order='K')
+
+ y = np.asarray(y, dtype=X.dtype)
+
+ if self.fit_intercept:
+ X_offset = np.average(X, axis=0)
+ X -= X_offset
+ if self.normalize:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ std = np.sqrt(np.sum(X**2, axis=0)/(len(X)-1))
+ X_scale[std != 0] = std[std != 0]
+ X /= X_scale
+ else:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ y_offset = np.mean(y)
+ y = y - y_offset
+ else:
+ X_offset = np.zeros(X.shape[1], dtype=X.dtype)
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ if y.ndim == 1:
+ y_offset = X.dtype.type(0)
+ else:
+ y_offset = np.zeros(y.shape[1], dtype=X.dtype)
+
+ return X, y, X_offset, y_offset, X_scale
+
+ def fit(self, X, y):
+ '''
+ Fits Regression with Orthogonal Matching Pursuit Algorithm.
+
+ Parameters
+ -----------
+ X: {array-like, sparse matrix} of size (n_samples, n_features)
+ Training data, matrix of explanatory variables
+
+ y: array-like of size [n_samples, n_features]
+ Target values
+
+ Returns
+ -------
+ self : object
+ Returns self.
+ '''
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._preprocess_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # Normalize columns of Psi, so that each column has norm = 1
+ norm_X = np.linalg.norm(X, axis=0)
+ X_norm = X/norm_X
+
+ # Initialize residual vector to full model response and normalize
+ R = y
+ norm_y = np.sqrt(np.dot(y, y))
+ r = y/norm_y
+
+ # Check for constant regressors
+ const_indices = np.where(~np.diff(X, axis=0).any(axis=0))[0]
+ bool_const = not const_indices
+
+ # Start regression using OPM algorithm
+ precision = 0 # Set precision criterion to precision of program
+ early_stop = True
+ cond_early = True # Initialize condition for early stop
+ ind = []
+ iindx = [] # index of selected columns
+ indtot = np.arange(n_features) # Full index set for remaining columns
+ kmax = min(n_samples, n_features) # Maximum number of iterations
+ LOO = np.PINF * np.ones(kmax) # Store LOO error at each iteration
+ LOOmin = np.PINF # Initialize minimum value of LOO
+ coeff = np.zeros((n_features, kmax))
+ count = 0
+ k = 0.1 # Percentage of iteration history for early stop
+
+ # Begin iteration over regressors set (Matrix X)
+ while (np.linalg.norm(R) > precision) and (count <= kmax-1) and \
+ ((cond_early or early_stop) ^ ~cond_early):
+
+ # Update index set of columns yet to select
+ if count != 0:
+ indtot = np.delete(indtot, iindx)
+
+ # Find column of X that is most correlated with residual
+ h = abs(np.dot(r, X_norm))
+ iindx = np.argmax(h[indtot])
+ indx = indtot[iindx]
+
+ # initialize with the constant regressor, if it exists in the basis
+ if (count == 0) and bool_const:
+ # overwrite values for iindx and indx
+ iindx = const_indices[0]
+ indx = indtot[iindx]
+
+ # Invert the information matrix at the first iteration, later only
+ # update its value on the basis of the previously inverted one,
+ if count == 0:
+ M = 1 / np.dot(X[:, indx], X[:, indx])
+ else:
+ x = np.dot(X[:, ind].T, X[:, indx])
+ r = np.dot(X[:, indx], X[:, indx])
+ M = self.blockwise_inverse(M, x, x.T, r)
+
+ # Add newly found index to the selected indexes set
+ ind.append(indx)
+
+ # Select regressors subset (Projection subspace)
+ Xpro = X[:, ind]
+
+ # Obtain coefficient by performing OLS
+ TT = np.dot(y, Xpro)
+ beta = np.dot(M, TT)
+ coeff[ind, count] = beta
+
+ # Compute LOO error
+ LOO[count] = self.loo_error(Xpro, M, y, beta)
+
+ # Compute new residual due to new projection
+ R = y - np.dot(Xpro, beta)
+
+ # Normalize residual
+ norm_R = np.sqrt(np.dot(R, R))
+ r = R / norm_R
+
+ # Update counters and early-stop criterions
+ countinf = max(0, int(count-k*kmax))
+ LOOmin = min(LOOmin, LOO[count])
+
+ if count == 0:
+ cond_early = (LOO[0] <= LOOmin)
+ else:
+ cond_early = (min(LOO[countinf:count+1]) <= LOOmin)
+
+ if self.verbose:
+ print(f'Iteration: {count+1}, mod. LOOCV error : '
+ f'{LOO[count]:.2e}')
+
+ # Update counter
+ count += 1
+
+ # Select projection with smallest cross-validation error
+ countmin = np.argmin(LOO[:-1])
+ self.coef_ = coeff[:, countmin]
+ self.active = coeff[:, countmin] != 0.0
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self, X):
+ '''
+ Computes predictive distribution for test set.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+ '''
+
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ return y_hat
+
+ def loo_error(self, psi, inv_inf_matrix, y, coeffs):
+ """
+ Calculates the corrected LOO error for regression on regressor
+ matrix `psi` that generated the coefficients based on [1] and [2].
+
+ [1] Blatman, G., 2009. Adaptive sparse polynomial chaos expansions for
+ uncertainty propagation and sensitivity analysis (Doctoral
+ dissertation, Clermont-Ferrand 2).
+
+ [2] Blatman, G. and Sudret, B., 2011. Adaptive sparse polynomial chaos
+ expansion based on least angle regression. Journal of computational
+ Physics, 230(6), pp.2345-2367.
+
+ Parameters
+ ----------
+ psi : array of shape (n_samples, n_feature)
+ Orthogonal bases evaluated at the samples.
+ inv_inf_matrix : array
+ Inverse of the information matrix.
+ y : array of shape (n_samples, )
+ Targets.
+ coeffs : array
+ Computed regresssor cofficients.
+
+ Returns
+ -------
+ loo_error : float
+ Modified LOOCV error.
+
+ """
+
+ # NrEvaluation (Size of experimental design)
+ N, P = psi.shape
+
+ # h factor (the full matrix is not calculated explicitly,
+ # only the trace is, to save memory)
+ PsiM = np.dot(psi, inv_inf_matrix)
+
+ h = np.sum(np.multiply(PsiM, psi), axis=1, dtype=np.longdouble)
+
+ # ------ Calculate Error Loocv for each measurement point ----
+ # Residuals
+ residual = np.dot(psi, coeffs) - y
+
+ # Variance
+ varY = np.var(y)
+
+ if varY == 0:
+ norm_emp_error = 0
+ loo_error = 0
+ else:
+ norm_emp_error = np.mean(residual**2)/varY
+
+ loo_error = np.mean(np.square(residual / (1-h))) / varY
+
+ # if there are NaNs, just return an infinite LOO error (this
+ # happens, e.g., when a strongly underdetermined problem is solved)
+ if np.isnan(loo_error):
+ loo_error = np.inf
+
+ # Corrected Error for over-determined system
+ tr_M = np.trace(np.atleast_2d(inv_inf_matrix))
+ if tr_M < 0 or abs(tr_M) > 1e6:
+ tr_M = np.trace(np.linalg.pinv(np.dot(psi.T, psi)))
+
+ # Over-determined system of Equation
+ if N > P:
+ T_factor = N/(N-P) * (1 + tr_M)
+
+ # Under-determined system of Equation
+ else:
+ T_factor = np.inf
+
+ loo_error *= T_factor
+
+ return loo_error
+
+ def blockwise_inverse(self, Ainv, B, C, D):
+ """
+ non-singular square matrix M defined as M = [[A B]; [C D]] .
+ B, C and D can have any dimension, provided their combination defines
+ a square matrix M.
+
+ Parameters
+ ----------
+ Ainv : float or array
+ inverse of the square-submatrix A.
+ B : float or array
+ Information matrix with all new regressor.
+ C : float or array
+ Transpose of B.
+ D : float or array
+ Information matrix with all selected regressors.
+
+ Returns
+ -------
+ M : array
+ Inverse of the information matrix.
+
+ """
+ if np.isscalar(D):
+ # Inverse of D
+ Dinv = 1/D
+ # Schur complement
+ SCinv = 1/(D - np.dot(C, np.dot(Ainv, B[:, None])))[0]
+ else:
+ # Inverse of D
+ Dinv = np.linalg.solve(D, np.eye(D.shape))
+ # Schur complement
+ SCinv = np.linalg.solve((D - C*Ainv*B), np.eye(D.shape))
+
+ T1 = np.dot(Ainv, np.dot(B[:, None], SCinv))
+ T2 = np.dot(C, Ainv)
+
+ # Assemble the inverse matrix
+ M = np.vstack((
+ np.hstack((Ainv+T1*T2, -T1)),
+ np.hstack((-(SCinv)*T2, SCinv))
+ ))
+ return M
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_ard.py b/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_ard.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6883a3edd6d247c219b8be328f5206b75780fbb
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_ard.py
@@ -0,0 +1,475 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar 24 19:41:45 2020
+
+@author: farid
+"""
+import numpy as np
+from scipy.linalg import solve_triangular
+from numpy.linalg import LinAlgError
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+import warnings
+from sklearn.utils import check_X_y
+from scipy.linalg import pinvh
+
+
+def update_precisions(Q,S,q,s,A,active,tol,n_samples,clf_bias):
+ '''
+ Selects one feature to be added/recomputed/deleted to model based on
+ effect it will have on value of log marginal likelihood.
+ '''
+ # initialise vector holding changes in log marginal likelihood
+ deltaL = np.zeros(Q.shape[0])
+
+ # identify features that can be added , recomputed and deleted in model
+ theta = q**2 - s
+ add = (theta > 0) * (active == False)
+ recompute = (theta > 0) * (active == True)
+ delete = ~(add + recompute)
+
+ # compute sparsity & quality parameters corresponding to features in
+ # three groups identified above
+ Qadd,Sadd = Q[add], S[add]
+ Qrec,Srec,Arec = Q[recompute], S[recompute], A[recompute]
+ Qdel,Sdel,Adel = Q[delete], S[delete], A[delete]
+
+ # compute new alpha's (precision parameters) for features that are
+ # currently in model and will be recomputed
+ Anew = s[recompute]**2/ ( theta[recompute] + np.finfo(np.float32).eps)
+ delta_alpha = (1./Anew - 1./Arec)
+
+ # compute change in log marginal likelihood
+ deltaL[add] = ( Qadd**2 - Sadd ) / Sadd + np.log(Sadd/Qadd**2 )
+ deltaL[recompute] = Qrec**2 / (Srec + 1. / delta_alpha) - np.log(1 + Srec*delta_alpha)
+ deltaL[delete] = Qdel**2 / (Sdel - Adel) - np.log(1 - Sdel / Adel)
+ deltaL = deltaL / n_samples
+
+ # find feature which caused largest change in likelihood
+ feature_index = np.argmax(deltaL)
+
+ # no deletions or additions
+ same_features = np.sum( theta[~recompute] > 0) == 0
+
+ # changes in precision for features already in model is below threshold
+ no_delta = np.sum( abs( Anew - Arec ) > tol ) == 0
+ # if same_features: print(abs( Anew - Arec ))
+ # print("same_features = {} no_delta = {}".format(same_features,no_delta))
+ # check convergence: if no features to add or delete and small change in
+ # precision for current features then terminate
+ converged = False
+ if same_features and no_delta:
+ converged = True
+ return [A,converged]
+
+ # if not converged update precision parameter of weights and return
+ if theta[feature_index] > 0:
+ A[feature_index] = s[feature_index]**2 / theta[feature_index]
+ if active[feature_index] == False:
+ active[feature_index] = True
+ else:
+ # at least two active features
+ if active[feature_index] == True and np.sum(active) >= 2:
+ # do not remove bias term in classification
+ # (in regression it is factored in through centering)
+ if not (feature_index == 0 and clf_bias):
+ active[feature_index] = False
+ A[feature_index] = np.PINF
+
+ return [A,converged]
+
+
+class RegressionFastARD(LinearModel, RegressorMixin):
+ '''
+ Regression with Automatic Relevance Determination (Fast Version uses
+ Sparse Bayesian Learning)
+ https://github.com/AmazaspShumik/sklearn-bayes/blob/master/skbayes/rvm_ard_models/fast_rvm.py
+
+ Parameters
+ ----------
+ n_iter: int, optional (DEFAULT = 100)
+ Maximum number of iterations
+
+ start: list, optional (DEFAULT = None)
+ Initial selected features.
+
+ tol: float, optional (DEFAULT = 1e-3)
+ If absolute change in precision parameter for weights is below threshold
+ algorithm terminates.
+
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ compute_score : bool, default=False
+ If True, compute the log marginal likelihood at each iteration of the
+ optimization.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ alpha_ : float
+ estimated precision of the noise
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ lambda_ : array, shape = (n_features)
+ estimated precisions of the coefficients
+
+ sigma_ : array, shape = (n_features, n_features)
+ estimated covariance matrix of the weights, computed only
+ for non-zero coefficients
+
+ scores_ : array-like of shape (n_iter_+1,)
+ If computed_score is True, value of the log marginal likelihood (to be
+ maximized) at each iteration of the optimization.
+
+ References
+ ----------
+ [1] Fast marginal likelihood maximisation for sparse Bayesian models
+ (Tipping & Faul 2003) (http://www.miketipping.com/papers/met-fastsbl.pdf)
+ [2] Analysis of sparse Bayesian learning (Tipping & Faul 2001)
+ (http://www.miketipping.com/abstracts.htm#Faul:NIPS01)
+ '''
+
+ def __init__(self, n_iter=300, start=None, tol=1e-3, fit_intercept=True,
+ normalize=False, copy_X=True, compute_score=False, verbose=False):
+ self.n_iter = n_iter
+ self.start = start
+ self.tol = tol
+ self.scores_ = list()
+ self.fit_intercept = fit_intercept
+ self.normalize = normalize
+ self.copy_X = copy_X
+ self.compute_score = compute_score
+ self.verbose = verbose
+
+ def _preprocess_data(self, X, y):
+ """Center and scale data.
+ Centers data to have mean zero along axis 0. If fit_intercept=False or
+ if the X is a sparse matrix, no centering is done, but normalization
+ can still be applied. The function returns the statistics necessary to
+ reconstruct the input data, which are X_offset, y_offset, X_scale, such
+ that the output
+ X = (X - X_offset) / X_scale
+ X_scale is the L2 norm of X - X_offset.
+ """
+
+ if self.copy_X:
+ X = X.copy(order='K')
+
+ y = np.asarray(y, dtype=X.dtype)
+
+ if self.fit_intercept:
+ X_offset = np.average(X, axis=0)
+ X -= X_offset
+ if self.normalize:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ std = np.sqrt(np.sum(X**2, axis=0)/(len(X)-1))
+ X_scale[std != 0] = std[std != 0]
+ X /= X_scale
+ else:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ y_offset = np.mean(y)
+ y = y - y_offset
+ else:
+ X_offset = np.zeros(X.shape[1], dtype=X.dtype)
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ if y.ndim == 1:
+ y_offset = X.dtype.type(0)
+ else:
+ y_offset = np.zeros(y.shape[1], dtype=X.dtype)
+
+ return X, y, X_offset, y_offset, X_scale
+
+ def fit(self, X, y):
+ '''
+ Fits ARD Regression with Sequential Sparse Bayes Algorithm.
+
+ Parameters
+ -----------
+ X: {array-like, sparse matrix} of size (n_samples, n_features)
+ Training data, matrix of explanatory variables
+
+ y: array-like of size [n_samples, n_features]
+ Target values
+
+ Returns
+ -------
+ self : object
+ Returns self.
+ '''
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._preprocess_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # precompute X'*Y , X'*X for faster iterations & allocate memory for
+ # sparsity & quality vectors
+ XY = np.dot(X.T, y)
+ XX = np.dot(X.T, X)
+ XXd = np.diag(XX)
+
+ # initialise precision of noise & and coefficients
+ var_y = np.var(y)
+
+ # check that variance is non zero !!!
+ if var_y == 0:
+ beta = 1e-2
+ self.var_y = True
+ else:
+ beta = 1. / np.var(y)
+ self.var_y = False
+
+ A = np.PINF * np.ones(n_features)
+ active = np.zeros(n_features, dtype=np.bool)
+
+ if self.start is not None and not hasattr(self, 'active_'):
+ start = self.start
+ # start from a given start basis vector
+ proj = XY**2 / XXd
+ active[start] = True
+ A[start] = XXd[start]/(proj[start] - var_y)
+
+ else:
+ # in case of almost perfect multicollinearity between some features
+ # start from feature 0
+ if np.sum(XXd - X_mean**2 < np.finfo(np.float32).eps) > 0:
+ A[0] = np.finfo(np.float16).eps
+ active[0] = True
+
+ else:
+ # start from a single basis vector with largest projection on
+ # targets
+ proj = XY**2 / XXd
+ start = np.argmax(proj)
+ active[start] = True
+ A[start] = XXd[start]/(proj[start] - var_y +
+ np.finfo(np.float32).eps)
+
+ warning_flag = 0
+ scores_ = []
+ for i in range(self.n_iter):
+ # Handle variance zero
+ if self.var_y:
+ A[0] = y_mean
+ active[0] = True
+ converged = True
+ break
+
+ XXa = XX[active, :][:, active]
+ XYa = XY[active]
+ Aa = A[active]
+
+ # mean & covariance of posterior distribution
+ Mn, Ri, cholesky = self._posterior_dist(Aa, beta, XXa, XYa)
+ if cholesky:
+ Sdiag = np.sum(Ri**2, 0)
+ else:
+ Sdiag = np.copy(np.diag(Ri))
+ warning_flag += 1
+
+ # raise warning in case cholesky fails
+ if warning_flag == 1:
+ warnings.warn(("Cholesky decomposition failed! Algorithm uses "
+ "pinvh, which is significantly slower. If you "
+ "use RVR it is advised to change parameters of "
+ "the kernel!"))
+
+ # compute quality & sparsity parameters
+ s, q, S, Q = self._sparsity_quality(XX, XXd, XY, XYa, Aa, Ri,
+ active, beta, cholesky)
+
+ # update precision parameter for noise distribution
+ rss = np.sum((y - np.dot(X[:, active], Mn))**2)
+
+ # if near perfect fit , then terminate
+ if (rss / n_samples/var_y) < self.tol:
+ warnings.warn('Early termination due to near perfect fit')
+ converged = True
+ break
+ beta = n_samples - np.sum(active) + np.sum(Aa * Sdiag)
+ beta /= rss
+ # beta /= (rss + np.finfo(np.float32).eps)
+
+ # update precision parameters of coefficients
+ A, converged = update_precisions(Q, S, q, s, A, active, self.tol,
+ n_samples, False)
+
+ if self.compute_score:
+ scores_.append(self.log_marginal_like(XXa, XYa, Aa, beta))
+
+ if self.verbose:
+ print(('Iteration: {0}, number of features '
+ 'in the model: {1}').format(i, np.sum(active)))
+
+ if converged or i == self.n_iter - 1:
+ if converged and self.verbose:
+ print('Algorithm converged!')
+ break
+
+ # after last update of alpha & beta update parameters
+ # of posterior distribution
+ XXa, XYa, Aa = XX[active, :][:, active], XY[active], A[active]
+ Mn, Sn, cholesky = self._posterior_dist(Aa, beta, XXa, XYa, True)
+ self.coef_ = np.zeros(n_features)
+ self.coef_[active] = Mn
+ self.sigma_ = Sn
+ self.active_ = active
+ self.lambda_ = A
+ self.alpha_ = beta
+ self.converged = converged
+ if self.compute_score:
+ self.scores_ = np.array(scores_)
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+ return self
+
+ def log_marginal_like(self, XXa, XYa, Aa, beta):
+ """Computes the log of the marginal likelihood."""
+ N, M = XXa.shape
+ A = np.diag(Aa)
+
+ Mn, sigma_, cholesky = self._posterior_dist(Aa, beta, XXa, XYa,
+ full_covar=True)
+
+ C = sigma_ + np.dot(np.dot(XXa.T, np.linalg.pinv(A)), XXa)
+
+ score = np.dot(np.dot(XYa.T, np.linalg.pinv(C)), XYa) +\
+ np.log(np.linalg.det(C)) + N * np.log(2 * np.pi)
+
+ return -0.5 * score
+
+ def predict(self, X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance based on
+ Ref.[1] Section 3.3.2.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ References
+ ----------
+ [1] Bishop, C. M. (2006). Pattern recognition and machine learning.
+ springer.
+ '''
+
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ if return_std:
+ # Handle the zero variance case
+ if self.var_y:
+ return y_hat, np.zeros_like(y_hat)
+
+ if self.normalize:
+ X -= self._x_mean_[self.active_]
+ X /= self._x_std[self.active_]
+ var_hat = 1./self.alpha_
+ var_hat += np.sum(X.dot(self.sigma_) * X, axis=1)
+ std_hat = np.sqrt(var_hat)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+ def _posterior_dist(self, A, beta, XX, XY, full_covar=False):
+ '''
+ Calculates mean and covariance matrix of posterior distribution
+ of coefficients.
+ '''
+ # compute precision matrix for active features
+ Sinv = beta * XX
+ np.fill_diagonal(Sinv, np.diag(Sinv) + A)
+ cholesky = True
+
+ # try cholesky, if it fails go back to pinvh
+ try:
+ # find posterior mean : R*R.T*mean = beta*X.T*Y
+ # solve(R*z = beta*X.T*Y) =>find z=> solve(R.T*mean = z)=>find mean
+ R = np.linalg.cholesky(Sinv)
+ Z = solve_triangular(R, beta*XY, check_finite=True, lower=True)
+ Mn = solve_triangular(R.T, Z, check_finite=True, lower=False)
+
+ # invert lower triangular matrix from cholesky decomposition
+ Ri = solve_triangular(R, np.eye(A.shape[0]), check_finite=False,
+ lower=True)
+ if full_covar:
+ Sn = np.dot(Ri.T, Ri)
+ return Mn, Sn, cholesky
+ else:
+ return Mn, Ri, cholesky
+ except LinAlgError:
+ cholesky = False
+ Sn = pinvh(Sinv)
+ Mn = beta*np.dot(Sinv, XY)
+ return Mn, Sn, cholesky
+
+ def _sparsity_quality(self, XX, XXd, XY, XYa, Aa, Ri, active, beta, cholesky):
+ '''
+ Calculates sparsity and quality parameters for each feature
+
+ Theoretical Note:
+ -----------------
+ Here we used Woodbury Identity for inverting covariance matrix
+ of target distribution
+ C = 1/beta + 1/alpha * X' * X
+ C^-1 = beta - beta^2 * X * Sn * X'
+ '''
+ bxy = beta*XY
+ bxx = beta*XXd
+ if cholesky:
+ # here Ri is inverse of lower triangular matrix obtained from
+ # cholesky decomp
+ xxr = np.dot(XX[:, active], Ri.T)
+ rxy = np.dot(Ri, XYa)
+ S = bxx - beta**2 * np.sum(xxr**2, axis=1)
+ Q = bxy - beta**2 * np.dot(xxr, rxy)
+ else:
+ # here Ri is covariance matrix
+ XXa = XX[:, active]
+ XS = np.dot(XXa, Ri)
+ S = bxx - beta**2 * np.sum(XS*XXa, 1)
+ Q = bxy - beta**2 * np.dot(XS, XYa)
+ # Use following:
+ # (EQ 1) q = A*Q/(A - S) ; s = A*S/(A-S)
+ # so if A = np.PINF q = Q, s = S
+ qi = np.copy(Q)
+ si = np.copy(S)
+ # If A is not np.PINF, then it should be 'active' feature => use (EQ 1)
+ Qa, Sa = Q[active], S[active]
+ qi[active] = Aa * Qa / (Aa - Sa)
+ si[active] = Aa * Sa / (Aa - Sa)
+
+ return [si, qi, S, Q]
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_laplace.py b/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_laplace.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fdcb5cf6e93c396d32eae2b0aad87a194a9cba4
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/reg_fast_laplace.py
@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import numpy as np
+from sklearn.utils import as_float_array
+from sklearn.model_selection import KFold
+
+
+class RegressionFastLaplace():
+ '''
+ Sparse regression with Bayesian Compressive Sensing as described in Alg. 1
+ (Fast Laplace) of Ref.[1], which updated formulas from [2].
+
+ sigma2: noise precision (sigma^2)
+ nu fixed to 0
+
+ uqlab/lib/uq_regression/BCS/uq_bsc.m
+
+ Parameters
+ ----------
+ n_iter: int, optional (DEFAULT = 1000)
+ Maximum number of iterations
+
+ tol: float, optional (DEFAULT = 1e-7)
+ If absolute change in precision parameter for weights is below
+ threshold algorithm terminates.
+
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ alpha_ : float
+ estimated precision of the noise
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ lambda_ : array, shape = (n_features)
+ estimated precisions of the coefficients
+
+ sigma_ : array, shape = (n_features, n_features)
+ estimated covariance matrix of the weights, computed only
+ for non-zero coefficients
+
+ References
+ ----------
+ [1] Babacan, S. D., Molina, R., & Katsaggelos, A. K. (2009). Bayesian
+ compressive sensing using Laplace priors. IEEE Transactions on image
+ processing, 19(1), 53-63.
+ [2] Fast marginal likelihood maximisation for sparse Bayesian models
+ (Tipping & Faul 2003).
+ (http://www.miketipping.com/papers/met-fastsbl.pdf)
+ '''
+
+ def __init__(self, n_iter=1000, n_Kfold=10, tol=1e-7, fit_intercept=False,
+ bias_term=True, copy_X=True, verbose=False):
+ self.n_iter = n_iter
+ self.n_Kfold = n_Kfold
+ self.tol = tol
+ self.fit_intercept = fit_intercept
+ self.bias_term = bias_term
+ self.copy_X = copy_X
+ self.verbose = verbose
+
+ def _center_data(self, X, y):
+ ''' Centers data'''
+ X = as_float_array(X, copy = self.copy_X)
+
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype=X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X, axis=0)
+ y_mean = np.average(y, axis=0)
+ X -= X_mean
+ y -= y_mean
+ else:
+ X_mean = np.zeros(X.shape[1], dtype=X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X, y, X_mean, y_mean, X_std
+
+ def fit(self, X, y):
+
+ k_fold = KFold(n_splits=self.n_Kfold)
+
+ varY = np.var(y, ddof=1) if np.var(y, ddof=1) != 0 else 1.0
+ sigma2s = len(y)*varY*(10**np.linspace(-16, -1, self.n_Kfold))
+
+ errors = np.zeros((len(sigma2s), self.n_Kfold))
+ for s, sigma2 in enumerate(sigma2s):
+ for k, (train, test) in enumerate(k_fold.split(X, y)):
+ self.fit_(X[train], y[train], sigma2)
+ errors[s, k] = np.linalg.norm(
+ y[test] - self.predict(X[test])
+ )**2/len(test)
+
+ KfCVerror = np.sum(errors, axis=1)/self.n_Kfold/varY
+ i_minCV = np.argmin(KfCVerror)
+
+ self.kfoldCVerror = np.min(KfCVerror)
+
+ return self.fit_(X, y, sigma2s[i_minCV])
+
+ def fit_(self, X, y, sigma2):
+
+ N, P = X.shape
+ # n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # check that variance is non zero !!!
+ if np.var(y) == 0:
+ self.var_y = True
+ else:
+ self.var_y = False
+ beta = 1./sigma2
+
+ # precompute X'*Y , X'*X for faster iterations & allocate memory for
+ # sparsity & quality vectors X=Psi
+ PsiTY = np.dot(X.T, y)
+ PsiTPsi = np.dot(X.T, X)
+ XXd = np.diag(PsiTPsi)
+
+ # initialize with constant regressor, or if that one does not exist,
+ # with the one that has the largest correlation with Y
+ ind_global_to_local = np.zeros(P, dtype=np.int32)
+
+ # identify constant regressors
+ constidx = np.where(~np.diff(X, axis=0).all(axis=0))[0]
+
+ if self.bias_term and constidx.size != 0:
+ ind_start = constidx[0]
+ ind_global_to_local[ind_start] = True
+ else:
+ # start from a single basis vector with largest projection on
+ # targets
+ proj = np.divide(np.square(PsiTY), XXd)
+ ind_start = np.argmax(proj)
+ ind_global_to_local[ind_start] = True
+
+ num_active = 1
+ active_indices = [ind_start]
+ deleted_indices = []
+ bcs_path = [ind_start]
+ gamma = np.zeros(P)
+ # for the initial value of gamma(ind_start), use the RVM formula
+ # gamma = (q^2 - s) / (s^2)
+ # and the fact that initially s = S = beta*Psi_i'*Psi_i and q = Q =
+ # beta*Psi_i'*Y
+ gamma[ind_start] = np.square(PsiTY[ind_start])
+ gamma[ind_start] -= sigma2 * PsiTPsi[ind_start, ind_start]
+ gamma[ind_start] /= np.square(PsiTPsi[ind_start, ind_start])
+
+ Sigma = 1. / (beta * PsiTPsi[ind_start, ind_start]
+ + 1./gamma[ind_start])
+
+ mu = Sigma * PsiTY[ind_start] * beta
+ tmp1 = beta * PsiTPsi[ind_start]
+ S = beta * np.diag(PsiTPsi).T - Sigma * np.square(tmp1)
+ Q = beta * PsiTY.T - mu*(tmp1)
+
+ tmp2 = np.ones(P) # alternative computation for the initial s,q
+ q0tilde = PsiTY[ind_start]
+ s0tilde = PsiTPsi[ind_start, ind_start]
+ tmp2[ind_start] = s0tilde / (q0tilde**2) / beta
+ s = np.divide(S, tmp2)
+ q = np.divide(Q, tmp2)
+ Lambda = 2*(num_active - 1) / np.sum(gamma)
+
+ Delta_L_max = []
+ for i in range(self.n_iter):
+ # Handle variance zero
+ if self.var_y:
+ mu = np.mean(y)
+ break
+
+ if self.verbose:
+ print(' lambda = {0:.6e}\n'.format(Lambda))
+
+ # Calculate the potential updated value of each gamma[i]
+ if Lambda == 0.0: # RVM
+ gamma_potential = np.multiply((
+ (q**2 - s) > Lambda),
+ np.divide(q**2 - s, s**2)
+ )
+ else:
+ a = Lambda * s**2
+ b = s**2 + 2*Lambda*s
+ c = Lambda + s - q**2
+ gamma_potential = np.multiply(
+ (c < 0), np.divide(
+ -b + np.sqrt(b**2 - 4*np.multiply(a, c)), 2*a)
+ )
+
+ l_gamma = - np.log(np.absolute(1 + np.multiply(gamma, s)))
+ l_gamma += np.divide(np.multiply(q**2, gamma),
+ (1 + np.multiply(gamma, s)))
+ l_gamma -= Lambda*gamma # omitted the factor 1/2
+
+ # Contribution of each updated gamma(i) to L(gamma)
+ l_gamma_potential = - np.log(
+ np.absolute(1 + np.multiply(gamma_potential, s))
+ )
+ l_gamma_potential += np.divide(
+ np.multiply(q**2, gamma_potential),
+ (1 + np.multiply(gamma_potential, s))
+ )
+ # omitted the factor 1/2
+ l_gamma_potential -= Lambda*gamma_potential
+
+ # Check how L(gamma) would change if we replaced gamma(i) by the
+ # updated gamma_potential(i), for each i separately
+ Delta_L_potential = l_gamma_potential - l_gamma
+
+ # deleted indices should not be chosen again
+ if len(deleted_indices) != 0:
+ values = -np.inf * np.ones(len(deleted_indices))
+ Delta_L_potential[deleted_indices] = values
+
+ Delta_L_max.append(np.nanmax(Delta_L_potential))
+ ind_L_max = np.nanargmax(Delta_L_potential)
+
+ # in case there is only 1 regressor in the model and it would now
+ # be deleted
+ if len(active_indices) == 1 and ind_L_max == active_indices[0] \
+ and gamma_potential[ind_L_max] == 0.0:
+ Delta_L_potential[ind_L_max] = -np.inf
+ Delta_L_max[i] = np.max(Delta_L_potential)
+ ind_L_max = np.argmax(Delta_L_potential)
+
+ # If L did not change significantly anymore, break
+ if Delta_L_max[i] <= 0.0 or\
+ (i > 0 and all(np.absolute(Delta_L_max[i-1:])
+ < sum(Delta_L_max)*self.tol)) or \
+ (i > 0 and all(np.diff(bcs_path)[i-1:] == 0.0)):
+ if self.verbose:
+ print('Increase in L: {0:.6e} (eta = {1:.3e})\
+ -- break\n'.format(Delta_L_max[i], self.tol))
+ break
+
+ # Print information
+ if self.verbose:
+ print(' Delta L = {0:.6e} \n'.format(Delta_L_max[i]))
+
+ what_changed = int(gamma[ind_L_max] == 0.0)
+ what_changed -= int(gamma_potential[ind_L_max] == 0.0)
+
+ # Print information
+ if self.verbose:
+ if what_changed < 0:
+ print(f'{i+1} - Remove regressor #{ind_L_max+1}..\n')
+ elif what_changed == 0:
+ print(f'{i+1} - Recompute regressor #{ind_L_max+1}..\n')
+ else:
+ print(f'{i+1} - Add regressor #{ind_L_max+1}..\n')
+
+ # --- Update all quantities ----
+ if what_changed == 1:
+ # adding a regressor
+
+ # update gamma
+ gamma[ind_L_max] = gamma_potential[ind_L_max]
+
+ Sigma_ii = 1.0 / (1.0/gamma[ind_L_max] + S[ind_L_max])
+ try:
+ x_i = np.matmul(
+ Sigma, PsiTPsi[active_indices, ind_L_max].reshape(-1, 1)
+ )
+ except ValueError:
+ x_i = Sigma * PsiTPsi[active_indices, ind_L_max]
+ tmp_1 = - (beta * Sigma_ii) * x_i
+ Sigma = np.vstack(
+ (np.hstack(((beta**2 * Sigma_ii) * np.dot(x_i, x_i.T)
+ + Sigma, tmp_1)), np.append(tmp_1.T, Sigma_ii))
+ )
+ mu_i = Sigma_ii * Q[ind_L_max]
+ mu = np.vstack((mu - (beta * mu_i) * x_i, mu_i))
+
+ tmp2_1 = PsiTPsi[:, ind_L_max] - beta * np.squeeze(
+ np.matmul(PsiTPsi[:, active_indices], x_i)
+ )
+ if i == 0:
+ tmp2_1[0] /= 2
+ tmp2 = beta * tmp2_1.T
+ S = S - Sigma_ii * np.square(tmp2)
+ Q = Q - mu_i * tmp2
+
+ num_active += 1
+ ind_global_to_local[ind_L_max] = num_active
+ active_indices.append(ind_L_max)
+ bcs_path.append(ind_L_max)
+
+ elif what_changed == 0:
+ # recomputation
+ # zero if regressor has not been chosen yet
+ if not ind_global_to_local[ind_L_max]:
+ raise Exception('Cannot recompute index{0} -- not yet\
+ part of the model!'.format(ind_L_max))
+ Sigma = np.atleast_2d(Sigma)
+ mu = np.atleast_2d(mu)
+ gamma_i_new = gamma_potential[ind_L_max]
+ gamma_i_old = gamma[ind_L_max]
+ # update gamma
+ gamma[ind_L_max] = gamma_potential[ind_L_max]
+
+ # index of regressor in Sigma
+ local_ind = ind_global_to_local[ind_L_max]-1
+
+ kappa_i = (1.0/gamma_i_new - 1.0/gamma_i_old)
+ kappa_i = 1.0 / kappa_i
+ kappa_i += Sigma[local_ind, local_ind]
+ kappa_i = 1 / kappa_i
+ Sigma_i_col = Sigma[:, local_ind]
+
+ Sigma = Sigma - kappa_i * (Sigma_i_col * Sigma_i_col.T)
+ mu_i = mu[local_ind]
+ mu = mu - (kappa_i * mu_i) * Sigma_i_col[:, None]
+
+ tmp1 = beta * np.dot(
+ Sigma_i_col.reshape(1, -1), PsiTPsi[active_indices])[0]
+ S = S + kappa_i * np.square(tmp1)
+ Q = Q + (kappa_i * mu_i) * tmp1
+
+ # no change in active_indices or ind_global_to_local
+ bcs_path.append(ind_L_max + 0.1)
+
+ elif what_changed == -1:
+ gamma[ind_L_max] = 0
+
+ # index of regressor in Sigma
+ local_ind = ind_global_to_local[ind_L_max]-1
+
+ Sigma_ii_inv = 1. / Sigma[local_ind, local_ind]
+ Sigma_i_col = Sigma[:, local_ind]
+
+ Sigma = Sigma - Sigma_ii_inv * (Sigma_i_col * Sigma_i_col.T)
+
+ Sigma = np.delete(
+ np.delete(Sigma, local_ind, axis=0), local_ind, axis=1)
+
+ mu = mu - (mu[local_ind] * Sigma_ii_inv) * Sigma_i_col[:, None]
+ mu = np.delete(mu, local_ind, axis=0)
+
+ tmp1 = beta * np.dot(Sigma_i_col, PsiTPsi[active_indices])
+ S = S + Sigma_ii_inv * np.square(tmp1)
+ Q = Q + (mu_i * Sigma_ii_inv) * tmp1
+
+ num_active -= 1
+ ind_global_to_local[ind_L_max] = 0.0
+ v = ind_global_to_local[ind_global_to_local > local_ind] - 1
+ ind_global_to_local[ind_global_to_local > local_ind] = v
+ del active_indices[local_ind]
+ deleted_indices.append(ind_L_max)
+ # and therefore ineligible
+ bcs_path.append(-ind_L_max)
+
+ # same for all three cases
+ tmp3 = 1 - np.multiply(gamma, S)
+ s = np.divide(S, tmp3)
+ q = np.divide(Q, tmp3)
+
+ # Update lambda
+ Lambda = 2*(num_active - 1) / np.sum(gamma)
+
+ # Prepare the result object
+ self.coef_ = np.zeros(P)
+ self.coef_[active_indices] = np.squeeze(mu)
+ self.sigma_ = Sigma
+ self.active_ = active_indices
+ self.gamma = gamma
+ self.Lambda = Lambda
+ self.beta = beta
+ self.bcs_path = bcs_path
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self, X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance based on
+ Ref.[1] Section 3.3.2.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+
+ References
+ ----------
+ [1] Bishop, C. M. (2006). Pattern recognition and machine learning.
+ springer.
+ '''
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ if return_std:
+ # Handle the zero variance case
+ if self.var_y:
+ return y_hat, np.zeros_like(y_hat)
+
+ var_hat = 1./self.beta
+ var_hat += np.sum(X.dot(self.sigma_) * X, axis=1)
+ std_hat = np.sqrt(var_hat)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+# l2norm = 0.0
+# for idx in range(10):
+# sigma2 = np.genfromtxt('./test/sigma2_{0}.csv'.format(idx+1), delimiter=',')
+# Psi_train = np.genfromtxt('./test/Psi_train_{0}.csv'.format(idx+1), delimiter=',')
+# Y_train = np.genfromtxt('./test/Y_train_{0}.csv'.format(idx+1))
+# Psi_test = np.genfromtxt('./test/Psi_test_{0}.csv'.format(idx+1), delimiter=',')
+# Y_test = np.genfromtxt('./test/Y_test_{0}.csv'.format(idx+1))
+
+# clf = RegressionFastLaplace(verbose=True)
+# clf.fit_(Psi_train, Y_train, sigma2)
+# coeffs_fold = np.genfromtxt('./test/coeffs_fold_{0}.csv'.format(idx+1))
+# print("coeffs error: {0:.4g}".format(np.linalg.norm(clf.coef_ - coeffs_fold)))
+# l2norm += np.linalg.norm(Y_test - clf.predict(Psi_test))**2/len(Y_test)
+# print("l2norm error: {0:.4g}".format(l2norm))
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/sequential_design.py b/examples/analytical-function/bayesvalidrox/surrogate_models/sequential_design.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc81dcd4529ca0708dfba47385aef4415992eb3e
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/sequential_design.py
@@ -0,0 +1,2187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 28 09:21:18 2022
+
+@author: farid
+"""
+import numpy as np
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from copy import deepcopy, copy
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import sys
+import os
+import gc
+import seaborn as sns
+from joblib import Parallel, delayed
+import resource
+from .exploration import Exploration
+
+
+class SeqDesign():
+ """ Sequential experimental design
+ This class provieds method for trainig the meta-model in an iterative
+ manners.
+ The main method to execute the task is `train_seq_design`, which
+ recieves a model object and returns the trained metamodel.
+ """
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self, MetaModel):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Parameters
+ ----------
+ Model : object
+ An object containing all model specifications.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ # MetaModel = self
+ Model = MetaModel.ModelObj
+ self.MetaModel = MetaModel
+ self.Model = Model
+
+ # Initialization
+ MetaModel.SeqModifiedLOO = {}
+ MetaModel.seqValidError = {}
+ MetaModel.SeqBME = {}
+ MetaModel.SeqKLD = {}
+ MetaModel.SeqDistHellinger = {}
+ MetaModel.seqRMSEMean = {}
+ MetaModel.seqRMSEStd = {}
+ MetaModel.seqMinDist = []
+ pce = True if MetaModel.meta_model_type.lower() != 'gpe' else False
+ mc_ref = True if bool(Model.mc_reference) else False
+ if mc_ref:
+ Model.read_mc_reference()
+
+ if not hasattr(MetaModel, 'valid_likelihoods'):
+ MetaModel.valid_samples = []
+ MetaModel.valid_model_runs = []
+ MetaModel.valid_likelihoods = []
+
+ # Get the parameters
+ max_n_samples = MetaModel.ExpDesign.n_max_samples
+ mod_LOO_threshold = MetaModel.ExpDesign.mod_LOO_threshold
+ n_canddidate = MetaModel.ExpDesign.n_canddidate
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ n_replication = MetaModel.ExpDesign.n_replication
+ util_func = MetaModel.ExpDesign.util_func
+ output_name = Model.Output.names
+ validError = None
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [MetaModel.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ if len(Model.observations) != 0 or Model.meas_file is not None:
+ self.observations = Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ TotalSigma2 = {}
+ # ---------- Initial MetaModel ----------
+ initMetaModel = deepcopy(MetaModel)
+
+ # Validation error if validation set is provided.
+ if len(MetaModel.valid_model_runs) != 0:
+ init_rmse, init_valid_error = self.__validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(MetaModel, 'Discrepancy'):
+ TotalSigma2 = MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self.__BME_Calculator(
+ initMetaModel, obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = MetaModel.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self.__error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean},"
+ f" {init_rmse_std}")
+
+ # Read the initial experimental design
+ Xinit = initMetaModel.ExpDesign.X
+ init_n_samples = len(MetaModel.ExpDesign.X)
+ initYprev = initMetaModel.ModelOutputDict
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = initMetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ initMetaModel.score_dict['b_1'][out_name].values()))
+ if MetaModel.dim_red_method.lower() == 'pca':
+ pca = MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ # Replicate the sequential design
+ for repIdx in range(n_replication):
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # To avoid changes ub original aPCE object
+ MetaModel.ExpDesign.X = Xinit
+ MetaModel.ExpDesign.Y = initYprev
+ MetaModel.LCerror = initLCerror
+
+ for util_f in util_func:
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ MetaModel.ExpDesign.X = Xinit
+ MetaModel.ExpDesign.Y = initYprev
+ MetaModel.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(MetaModel)
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel(
+ samples=Xfull[-1].reshape(1, -1))
+
+ # Optimal Bayesian Design
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.opt_SeqDesign(TotalSigma2,
+ n_canddidate,
+ util_f)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ MetaModel.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(MetaModel, 'adapt_verbose') and \
+ MetaModel.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = MetaModel.eval_metamodel(samples=Xnew)
+ adaptPlot(MetaModel, Ynew, y_hat, std_hat, plotED=False)
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ MetaModel.ModelOutputDict[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ MetaModel.ExpDesign.sampling_method = 'user'
+ MetaModel.ExpDesign.X = Xfull
+ MetaModel.ExpDesign.Y = MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = MetaModel.ModelOutputDict
+
+ # Pass the new prior as the input
+ MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ MetaModel.train_norm_design(parallel=False)
+ m_3 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ MetaModel.score_dict['b_1'][out_name].values()))
+ if MetaModel.dim_red_method.lower() == 'pca':
+ pca = MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if len(MetaModel.valid_model_runs) != 0:
+ rmse, validError = self.__validError(MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ m_4 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self.__BME_Calculator(MetaModel, obs_data,
+ TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = MetaModel.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = MetaModel.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+ m_5 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self.__error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ print(f"Memory itr {itr_no}: I: {m_2-m_1:.2f} MB")
+ print(f"Memory itr {itr_no}: II: {m_3-m_2:.2f} MB")
+ print(f"Memory itr {itr_no}: III: {m_4-m_3:.2f} MB")
+ print(f"Memory itr {itr_no}: IV: {m_5-m_4:.2f} MB")
+ m_6 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"Memory itr {itr_no}: total: {m_6:.2f} MB")
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ gc.collect()
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ MetaModel.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(MetaModel.valid_model_runs) != 0:
+ MetaModel.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ MetaModel.SeqBME[strKey] = SeqBME
+ MetaModel.SeqKLD[strKey] = SeqKLD
+ if len(MetaModel.valid_likelihoods) != 0:
+ MetaModel.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ MetaModel.seqRMSEMean[strKey] = seqRMSEMean
+ MetaModel.seqRMSEStd[strKey] = seqRMSEStd
+
+ return MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = MetaModel.ExpDesign.X
+ out_dict_y = MetaModel.ExpDesign.Y
+ out_names = MetaModel.ModelObj.Output.names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, X_can, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_mean_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Get the data
+ obs_data = self.observations
+ n_obs = self.Model.n_obs
+ # TODO: Analytical DKL
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+
+ # priorMean, priorSigma2, Obs = np.empty((0)),np.empty((0)),np.empty((0))
+
+ # for key in list(Y_mean_can):
+ # # concatenate the measurement error
+ # Obs = np.hstack((Obs,ObservationData[key]))
+
+ # # concatenate the mean and variance of prior predictive
+ # means, stds = Y_mean_can[key][0], Y_std_can[key][0]
+ # priorMean = np.hstack((priorSigma2,means))
+ # priorSigma2 = np.hstack((priorSigma2,stds**2))
+
+ # # Covariance Matrix of prior
+ # covPrior = np.zeros((priorSigma2.shape[0], priorSigma2.shape[0]), float)
+ # np.fill_diagonal(covPrior, priorSigma2)
+
+ # # Covariance Matrix of Likelihood
+ # covLikelihood = np.zeros((sigma2Dict.shape[0], sigma2Dict.shape[0]), float)
+ # np.fill_diagonal(covLikelihood, sigma2Dict)
+
+ # # Calculate moments of the posterior (Analytical derivation)
+ # n = priorSigma2.shape[0]
+ # covPost = np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))),covLikelihood/n)
+
+ # meanPost = np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))) , Obs) + \
+ # np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))),
+ # priorMean/n)
+ # # Compute DKL from prior to posterior
+ # term1 = np.trace(np.dot(np.linalg.inv(covPrior),covPost))
+ # deltaMean = priorMean-meanPost
+ # term2 = np.dot(np.dot(deltaMean,np.linalg.inv(covPrior)),deltaMean[:,None])
+ # term3 = np.log(np.linalg.det(covPrior)/np.linalg.det(covPost))
+ # DKL = 0.5 * (term1 + term2 - n + term3)[0]
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = 20000
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((MCsize))
+ for key in list(Y_mean_can):
+ means, stds = Y_mean_can[key][0], Y_std_can[key][0]
+ # cov = np.zeros((means.shape[0], means.shape[0]), float)
+ # np.fill_diagonal(cov, stds**2)
+
+ Y_MC[key] = np.zeros((MCsize, n_obs))
+ logsamples = np.zeros((MCsize, n_obs))
+ for i in range(n_obs):
+ NormalDensity = stats.norm(means[i], stds[i])
+ Y_MC[key][:, i] = NormalDensity.rvs(MCsize)
+ logsamples[:, i] = NormalDensity.logpdf(Y_MC[key][:, i])
+
+ logPriorLikelihoods = np.sum(logsamples, axis=1)
+ std_MC[key] = np.zeros((MCsize, means.shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.nansum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = logBME
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self.__normpdf(
+ Y_mean_can, Y_std_can, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+ gc.collect(generation=2)
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def update_metamodel(self, MetaModel, output, y_hat_can, univ_p_val, index,
+ new_pca=False):
+ BasisIndices = MetaModel.basis_dict[output]["y_"+str(index+1)]
+ clf_poly = MetaModel.clf_poly[output]["y_"+str(index+1)]
+ Mn = clf_poly.coef_
+ Sn = clf_poly.sigma_
+ beta = clf_poly.alpha_
+ active = clf_poly.active_
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ Sn_new_inv = np.linalg.inv(Sn)
+ Sn_new_inv += beta * np.dot(Psi[:, active].T, Psi[:, active])
+ Sn_new = np.linalg.inv(Sn_new_inv)
+
+ Mn_new = np.dot(Sn_new_inv, Mn[active]).reshape(-1, 1)
+ Mn_new += beta * np.dot(Psi[:, active].T, y_hat_can)
+ Mn_new = np.dot(Sn_new, Mn_new).flatten()
+
+ # Compute the old and new moments of PCEs
+ mean_old = Mn[0]
+ mean_new = Mn_new[0]
+ std_old = np.sqrt(np.sum(np.square(Mn[1:])))
+ std_new = np.sqrt(np.sum(np.square(Mn_new[1:])))
+
+ # Back transformation if PCA is selected.
+ if MetaModel.dim_red_method.lower() == 'pca':
+ old_pca = MetaModel.pca[output]
+ mean_old = old_pca.mean_[index]
+ mean_old += np.sum(mean_old * old_pca.components_[:, index])
+ std_old = np.sqrt(np.sum(std_old**2 *
+ old_pca.components_[:, index]**2))
+ mean_new = new_pca.mean_[index]
+ mean_new += np.sum(mean_new * new_pca.components_[:, index])
+ std_new = np.sqrt(np.sum(std_new**2 *
+ new_pca.components_[:, index]**2))
+ # print(f"mean_old: {mean_old:.2f} mean_new: {mean_new:.2f}")
+ # print(f"std_old: {std_old:.2f} std_new: {std_new:.2f}")
+ # Store the old and new moments of PCEs
+ results = {
+ 'mean_old': mean_old,
+ 'mean_new': mean_new,
+ 'std_old': std_old,
+ 'std_new': std_new
+ }
+ return results
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign_old(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ old_EDY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels using the candidate design
+ Y_PC_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Generate y from posterior predictive
+ m_size = 100
+ y_hat_samples = {}
+ for idx, key in enumerate(Model.Output.names):
+ means, stds = Y_PC_can[key][0], Y_std_can[key][0]
+ y_hat_samples[key] = np.random.multivariate_normal(
+ means, np.diag(stds), m_size)
+
+ # Create the SparseBayes-based PCE metamodel:
+ MetaModel.input_obj.poly_coeffs_flag = False
+ univ_p_val = self.MetaModel.univ_basis_vals(X_can)
+ G_n_m_all = np.zeros((m_size, len(Model.Output.names), Model.n_obs))
+
+ for i in range(m_size):
+ for idx, key in enumerate(Model.Output.names):
+ if MetaModel.dim_red_method.lower() == 'pca':
+ # Equal number of components
+ new_outputs = np.vstack(
+ (old_EDY[key], y_hat_samples[key][i])
+ )
+ new_pca, _ = MetaModel.pca_transformation(new_outputs)
+ target = new_pca.transform(
+ y_hat_samples[key][i].reshape(1, -1)
+ )[0]
+ else:
+ new_pca, target = False, y_hat_samples[key][i]
+
+ for j in range(len(target)):
+
+ # Update surrogate
+ result = self.update_metamodel(
+ MetaModel, key, target[j], univ_p_val, j, new_pca)
+
+ # Compute Expected Information Gain (Eq. 39)
+ G_n_m = np.log(result['std_old']/result['std_new']) - 1./2
+ G_n_m += result['std_new']**2 / (2*result['std_old']**2)
+ G_n_m += (result['mean_new'] - result['mean_old'])**2 /\
+ (2*result['std_old']**2)
+
+ G_n_m_all[i, idx, j] = G_n_m
+
+ U_J_d = G_n_m_all.mean(axis=(1, 2)).mean()
+ return -1 * U_J_d
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ out_names = MetaModel.ModelObj.Output.names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+ oldExpDesignY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+
+ # Add all suggestion as new ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ try:
+ NewExpDesignY[key] = np.vstack((oldExpDesignY[key],
+ Y_PC_can[key]))
+ except:
+ NewExpDesignY[key] = oldExpDesignY[key]
+
+ MetaModel.ExpDesign.sampling_method = 'user'
+ MetaModel.ExpDesign.X = NewExpDesignX
+ MetaModel.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ MetaModel.input_obj.poly_coeffs_flag = False
+ MetaModel.train_norm_design(parallel=False)
+ PCE_Model_can = MetaModel
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = PCE_Model_can.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.MetaModel.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.float128))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted] / np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+ gc.collect(generation=2)
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def subdomain(self, Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ Subdomains : TYPE
+ DESCRIPTION.
+
+ """
+ n_params = self.MetaModel.n_params
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianActiveDesign(X_can, sigma2Dict,
+ var)
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithim to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.MetaModel.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def opt_SeqDesign(self, sigma2, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None.
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ MetaModel = self.MetaModel
+ Bounds = MetaModel.bound_tuples
+ n_new_samples = MetaModel.ExpDesign.n_new_samples
+ explore_method = MetaModel.ExpDesign.explore_method
+ exploit_method = MetaModel.ExpDesign.exploit_method
+ n_cand_groups = MetaModel.ExpDesign.n_cand_groups
+ tradeoff_scheme = MetaModel.ExpDesign.tradeoff_scheme
+
+ old_EDX = MetaModel.ExpDesign.X
+ old_EDY = MetaModel.ExpDesign.Y.copy()
+ ndim = MetaModel.ExpDesign.X.shape[1]
+ OutputNames = MetaModel.ModelObj.Output.names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+
+ Xnew, filteredSamples = MetaModel.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ args = []
+ subdomains = self.subdomain(Bounds, n_new_samples)
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+
+ print("\nXnew:\n", Xnew)
+
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ elif explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = MetaModel.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(MetaModel, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = MetaModel.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = MetaModel.ExpDesign.generate_samples(
+ MetaModel.ExpDesign.max_func_itr, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+
+ results = Parallel(n_jobs=-1, backend='threading')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ # out = map(self.run_util_func,
+ # [exploit_method]*n_cand_groups,
+ # split_cand,
+ # range(n_cand_groups),
+ # [sigma2] * n_cand_groups,
+ # [var] * n_cand_groups,
+ # [X_MC] * n_cand_groups
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # create surrogate model for U_J_d
+ from sklearn.preprocessing import MinMaxScaler
+ # Take care of inf entries
+ good_indices = [i for i, arr in enumerate(U_J_d)
+ if np.isfinite(arr).all()]
+ scaler = MinMaxScaler()
+ X_S = scaler.fit_transform(candidates[good_indices])
+ gp = MetaModel.gaussian_process_emulator(
+ X_S, U_J_d[good_indices], autoSelect=True
+ )
+ U_J_d = gp.predict(scaler.transform(allCandidates))
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ print("norm_U_J_d:\n", norm_U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Total score
+ totalScore = exploit_w * norm_U_J_d
+ totalScore += explore_w * scoreExploration
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+ # out = map(self.run_util_func,
+ # [exploit_method]*len(goodSampleIdx),
+ # split_cand,
+ # range(len(goodSampleIdx)),
+ # [sigma2] * len(goodSampleIdx),
+ # [var] * len(goodSampleIdx)
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+ gc.collect()
+
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self
+ Model = self.Model
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = Model.Output.names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ Scores = list(MetaModel.score_dict[OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ BasisIndices = MetaModelOrig.basis_dict[OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = MetaModelOrig.univ_basis_vals(oldExpDesignX)
+ Psi = MetaModelOrig.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.univ_basis_vals(candidates)
+ Psi_c = self.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var == 'D-Opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var == 'A-Opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var == 'K-Opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def __normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+
+ Model = self.Model
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(Model.Output.names):
+
+ # (Meta)Model Output
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = MetaModel.ExpDesign.X # valid_samples
+ model_outputs = MetaModel.ExpDesign.Y # valid_model_runs
+ Model = MetaModel.ModelObj
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self.__logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self.__logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def __logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def __posteriorPlot(self, posterior, par_names, key):
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.MetaModel.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+ # -------------------------------------------------------------------------
+ def __hellinger_distance(self, P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+ # -------------------------------------------------------------------------
+ def __BME_Calculator(self, MetaModel, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ """
+ # Initializations
+ valid_likelihoods = MetaModel.valid_likelihoods
+
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ if post_snapshot or len(valid_likelihoods) != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = MetaModel.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ Y_MC, std_MC = MetaModel.eval_metamodel(samples=X_MC)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"\nMemory eval_metamodel in BME: {m_2-m_1:.2f} MB")
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self.__normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(MetaModel.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or len(valid_likelihoods) != 0:
+ # Hellinger distance
+ ref_like = np.log(valid_likelihoods[valid_likelihoods > 0])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = self.__hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and MetaModel.n_params == 2 and not idx % 5:
+ from bayes_inference.bayes_inference import BayesInference
+ from bayes_inference.discrepancy import Discrepancy
+ import pandas as pd
+ BayesOpts = BayesInference(MetaModel)
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ obs_data = pd.DataFrame(obs_data, columns=self.Model.Output.names)
+ BayesOpts.measurement_error = obs_data
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ # Clean up
+ del Y_MC, std_MC
+ gc.collect()
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def __validError(self, MetaModel):
+
+ # MetaModel = self.MetaModel
+ Model = MetaModel.ModelObj
+ OutputName = Model.Output.names
+
+ # Extract the original model with the generated samples
+ valid_samples = MetaModel.valid_samples
+ valid_model_runs = MetaModel.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ valid_PCE_runs, valid_PCE_std = MetaModel.eval_metamodel(samples=valid_samples)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"\nMemory eval_metamodel: {m_2-m_1:.2f} MB")
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in OutputName:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def __error_Mean_Std(self):
+
+ MetaModel = self.MetaModel
+ # Extract the mean and std provided by user
+ df_MCReference = MetaModel.ModelObj.mc_reference
+
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+
+ # Compute the root mean squared error
+ for output in MetaModel.ModelObj.Output.names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ df_MCReference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ df_MCReference['std'], pce_means[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self, MetaModel):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+ outputs = MetaModel.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.mean_ + np.dot(
+ pce_mean, PCA.components_)
+ stds[output] = np.sqrt(np.dot(pce_var,
+ PCA.components_**2))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/examples/analytical-function/bayesvalidrox/surrogate_models/surrogate_models.py b/examples/analytical-function/bayesvalidrox/surrogate_models/surrogate_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..e318dfc32a22a1b02efc4ee8d3ccbaa4b2b190f3
--- /dev/null
+++ b/examples/analytical-function/bayesvalidrox/surrogate_models/surrogate_models.py
@@ -0,0 +1,1581 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Implementation of metamodel as either PC, aPC or GPE
+"""
+
+import warnings
+import numpy as np
+import math
+import h5py
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import MinMaxScaler
+import scipy as sp
+from scipy.optimize import minimize, NonlinearConstraint, LinearConstraint
+from tqdm import tqdm
+from sklearn.decomposition import PCA as sklearnPCA
+import sklearn.linear_model as lm
+from sklearn.gaussian_process import GaussianProcessRegressor
+import sklearn.gaussian_process.kernels as kernels
+import os
+from joblib import Parallel, delayed
+import copy
+
+from .input_space import InputSpace
+from .glexindex import glexindex
+from .eval_rec_rule import eval_univ_basis
+from .reg_fast_ard import RegressionFastARD
+from .reg_fast_laplace import RegressionFastLaplace
+from .orthogonal_matching_pursuit import OrthogonalMatchingPursuit
+from .bayes_linear import VBLinearRegression, EBLinearRegression
+from .apoly_construction import apoly_construction
+warnings.filterwarnings("ignore")
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class MetaModel():
+ """
+ Meta (surrogate) model
+
+ This class trains a surrogate model. It accepts an input object (input_obj)
+ containing the specification of the distributions for uncertain parameters
+ and a model object with instructions on how to run the computational model.
+
+ Attributes
+ ----------
+ input_obj : obj
+ Input object with the information on the model input parameters.
+ meta_model_type : str
+ Surrogate model types. Three surrogate model types are supported:
+ polynomial chaos expansion (`PCE`), arbitrary PCE (`aPCE`) and
+ Gaussian process regression (`GPE`). Default is PCE.
+ pce_reg_method : str
+ PCE regression method to compute the coefficients. The following
+ regression methods are available:
+
+ 1. OLS: Ordinary Least Square method
+ 2. BRR: Bayesian Ridge Regression
+ 3. LARS: Least angle regression
+ 4. ARD: Bayesian ARD Regression
+ 5. FastARD: Fast Bayesian ARD Regression
+ 6. VBL: Variational Bayesian Learning
+ 7. EBL: Emperical Bayesian Learning
+ Default is `OLS`.
+ bootstrap_method : str
+ Bootstraping method. Options are `'normal'` and `'fast'`. The default
+ is `'fast'`. It means that in each iteration except the first one, only
+ the coefficent are recalculated with the ordinary least square method.
+ n_bootstrap_itrs : int
+ Number of iterations for the bootstrap sampling. The default is `1`.
+ pce_deg : int or list of int
+ Polynomial degree(s). If a list is given, an adaptive algorithm is used
+ to find the best degree with the lowest Leave-One-Out cross-validation
+ (LOO) error (or the highest score=1-LOO). Default is `1`.
+ pce_q_norm : float
+ Hyperbolic (or q-norm) truncation for multi-indices of multivariate
+ polynomials. Default is `1.0`.
+ dim_red_method : str
+ Dimensionality reduction method for the output space. The available
+ method is based on principal component analysis (PCA). The Default is
+ `'no'`. There are two ways to select number of components: use
+ percentage of the explainable variance threshold (between 0 and 100)
+ (Option A) or direct prescription of components' number (Option B):
+
+ >>> MetaModelOpts.dim_red_method = 'PCA'
+ >>> MetaModelOpts.var_pca_threshold = 99.999 # Option A
+ >>> MetaModelOpts.n_pca_components = 12 # Option B
+ apply_constraints : bool
+ If set to true constraints will be applied during training.
+ In this case the training uses OLS. In this version the constraints
+ need to be set explicitly in this class.
+
+ verbose : bool
+ Prints summary of the regression results. Default is `False`.
+
+ Note
+ -------
+ To define the sampling methods and the training set, an experimental design
+ instance shall be defined. This can be done by:
+
+ >>> MetaModelOpts.add_InputSpace()
+
+ Two experimental design schemes are supported: one-shot (`normal`) and
+ adaptive sequential (`sequential`) designs.
+ For experimental design refer to `InputSpace`.
+
+ """
+
+ def __init__(self, input_obj, meta_model_type='PCE',
+ pce_reg_method='OLS', bootstrap_method='fast',
+ n_bootstrap_itrs=1, pce_deg=1, pce_q_norm=1.0,
+ dim_red_method='no', apply_constraints = False,
+ verbose=False):
+
+ self.input_obj = input_obj
+ self.meta_model_type = meta_model_type
+ self.pce_reg_method = pce_reg_method
+ self.bootstrap_method = bootstrap_method
+ self.n_bootstrap_itrs = n_bootstrap_itrs
+ self.pce_deg = pce_deg
+ self.pce_q_norm = pce_q_norm
+ self.dim_red_method = dim_red_method
+ self.apply_constraints = apply_constraints
+ self.verbose = verbose
+
+ def build_metamodel(self, n_init_samples = None) -> None:
+ """
+ Builds the parts for the metamodel (polynomes,...) that are neede before fitting.
+
+ Returns
+ -------
+ None
+ DESCRIPTION.
+
+ """
+
+ # Generate general warnings
+ if self.apply_constraints or self.pce_reg_method.lower() == 'ols':
+ print('There are no estimations of surrogate uncertainty available'
+ ' for the chosen regression options. This might lead to issues'
+ ' in later steps.')
+
+ # Add InputSpace to MetaModel if it does not have any
+ if not hasattr(self, 'InputSpace'):
+ self.InputSpace = InputSpace(self.input_obj)
+ self.InputSpace.n_init_samples = n_init_samples
+ self.InputSpace.init_param_space(np.max(self.pce_deg))
+
+ self.ndim = self.InputSpace.ndim
+
+ if not hasattr(self, 'CollocationPoints'):
+ raise AttributeError('Please provide samples to the metamodel before building it.')
+
+ # Transform input samples
+ # TODO: this is probably not yet correct! Make 'method' variable
+ self.CollocationPoints = self.InputSpace.transform(self.CollocationPoints, method='user')
+
+
+ self.n_params = len(self.input_obj.Marginals)
+
+ # Generate polynomials
+ if self.meta_model_type.lower() != 'gpe':
+ self.generate_polynomials(np.max(self.pce_deg))
+
+ # Initialize the nested dictionaries
+ if self.meta_model_type.lower() == 'gpe':
+ self.gp_poly = self.auto_vivification()
+ self.x_scaler = self.auto_vivification()
+ self.LCerror = self.auto_vivification()
+ else:
+ self.deg_dict = self.auto_vivification()
+ self.q_norm_dict = self.auto_vivification()
+ self.coeffs_dict = self.auto_vivification()
+ self.basis_dict = self.auto_vivification()
+ self.score_dict = self.auto_vivification()
+ self.clf_poly = self.auto_vivification()
+ self.LCerror = self.auto_vivification()
+ if self.dim_red_method.lower() == 'pca':
+ self.pca = self.auto_vivification()
+
+ # Define an array containing the degrees
+ self.CollocationPoints = np.array(self.CollocationPoints)
+ self.n_samples, ndim = self.CollocationPoints.shape
+ if self.ndim != ndim:
+ raise AttributeError('The given samples do not match the given number of priors. The samples should be a 2D array of size (#samples, #priors)')
+
+ self.deg_array = self.__select_degree(ndim, self.n_samples)
+
+ # Generate all basis indices
+ self.allBasisIndices = self.auto_vivification()
+ for deg in self.deg_array:
+ keys = self.allBasisIndices.keys()
+ if deg not in np.fromiter(keys, dtype=float):
+ # Generate the polynomial basis indices
+ for qidx, q in enumerate(self.pce_q_norm):
+ basis_indices = glexindex(start=0, stop=deg+1,
+ dimensions=self.n_params,
+ cross_truncation=q,
+ reverse=False, graded=True)
+ self.allBasisIndices[str(deg)][str(q)] = basis_indices
+
+
+
+ def fit(self, X, y, parallel = True, verbose = False):
+ """
+ Fits the surrogate to the given data (samples X, outputs y).
+ Note here that the samples X should be the transformed samples provided
+ by the experimental design if the transformation is used there.
+
+ Parameters
+ ----------
+ X : 2D list or np.array of shape (#samples, #dim)
+ The parameter value combinations that the model was evaluated at.
+ y : dict of 2D lists or arrays of shape (#samples, #timesteps)
+ The respective model evaluations.
+
+ Returns
+ -------
+ None.
+
+ """
+# print(X)
+# print(X.shape)
+# print(y)
+# print(y['Z'].shape)
+ X = np.array(X)
+ for key in y.keys():
+ y_val = np.array(y[key])
+ if y_val.ndim !=2:
+ raise ValueError('The given outputs y should be 2D')
+ y[key] = np.array(y[key])
+
+ # Output names are the same as the keys in y
+ self.out_names = list(y.keys())
+
+ # Build the MetaModel on the static samples
+ self.CollocationPoints = X
+
+ # TODO: other option: rebuild every time
+ if not hasattr(self, 'deg_array'):
+ self.build_metamodel(n_init_samples = X.shape[1])
+
+ # Evaluate the univariate polynomials on InputSpace
+ if self.meta_model_type.lower() != 'gpe':
+ self.univ_p_val = self.univ_basis_vals(self.CollocationPoints)
+
+ # --- Loop through data points and fit the surrogate ---
+ if verbose:
+ print(f"\n>>>> Training the {self.meta_model_type} metamodel "
+ "started. <<<<<<\n")
+
+ # --- Bootstrap sampling ---
+ # Correct number of bootstrap if PCA transformation is required.
+ if self.dim_red_method.lower() == 'pca' and self.n_bootstrap_itrs == 1:
+ self.n_bootstrap_itrs = 100
+
+ # Check if fast version (update coeffs with OLS) is selected.
+ if self.bootstrap_method.lower() == 'fast':
+ fast_bootstrap = True
+ first_out = {}
+ n_comp_dict = {}
+ else:
+ fast_bootstrap = False
+
+ # Prepare tqdm iteration maessage
+ if verbose and self.n_bootstrap_itrs > 1:
+ enum_obj = tqdm(range(self.n_bootstrap_itrs),
+ total=self.n_bootstrap_itrs,
+ desc="Bootstrapping the metamodel",
+ ascii=True)
+ else:
+ enum_obj = range(self.n_bootstrap_itrs)
+
+ # Loop over the bootstrap iterations
+ for b_i in enum_obj:
+ if b_i > 0:
+ b_indices = np.random.randint(self.n_samples, size=self.n_samples)
+ else:
+ b_indices = np.arange(len(X))
+
+ X_train_b = X[b_indices]
+
+ if verbose and self.n_bootstrap_itrs == 1:
+ items = tqdm(y.items(), desc="Fitting regression")
+ else:
+ items = y.items()
+
+ # For loop over the components/outputs
+ for key, Output in items:
+
+ # Dimensionality reduction with PCA, if specified
+ if self.dim_red_method.lower() == 'pca':
+
+ # Use the stored n_comp for fast bootsrtrapping
+ if fast_bootstrap and b_i > 0:
+ self.n_pca_components = n_comp_dict[key]
+
+ # Start transformation
+ pca, target, n_comp = self.pca_transformation(
+ Output[b_indices], verbose=False
+ )
+ self.pca[f'b_{b_i+1}'][key] = pca
+ # Store the number of components for fast bootsrtrapping
+ if fast_bootstrap and b_i == 0:
+ n_comp_dict[key] = n_comp
+ else:
+ #print(b_indices)
+ target = Output[b_indices]
+
+ # Parallel fit regression
+ if self.meta_model_type.lower() == 'gpe':
+ # Prepare the input matrix
+ scaler = MinMaxScaler()
+ X_S = scaler.fit_transform(X_train_b)
+
+ self.x_scaler[f'b_{b_i+1}'][key] = scaler
+ if parallel:
+ out = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.gaussian_process_emulator)(
+ X_S, target[:, idx]) for idx in
+ range(target.shape[1]))
+ else:
+ results = map(self.gaussian_process_emulator,
+ [X_train_b]*target.shape[1],
+ [target[:, idx] for idx in
+ range(target.shape[1])]
+ )
+ out = list(results)
+
+ for idx in range(target.shape[1]):
+ self.gp_poly[f'b_{b_i+1}'][key][f"y_{idx+1}"] = out[idx]
+
+ else:
+ self.univ_p_val = self.univ_p_val[b_indices]
+ if parallel and (not fast_bootstrap or b_i == 0):
+ out = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.adaptive_regression)(X_train_b,
+ target[:, idx],
+ idx)
+ for idx in range(target.shape[1]))
+ elif not parallel and (not fast_bootstrap or b_i == 0):
+ results = map(self.adaptive_regression,
+ [X_train_b]*target.shape[1],
+ [target[:, idx] for idx in
+ range(target.shape[1])],
+ range(target.shape[1]))
+ out = list(results)
+
+ # Store the first out dictionary
+ if fast_bootstrap and b_i == 0:
+ first_out[key] = copy.deepcopy(out)
+
+ if b_i > 0 and fast_bootstrap:
+
+ # fast bootstrap
+ out = self.update_pce_coeffs(
+ X_train_b, target, first_out[key])
+
+ for i in range(target.shape[1]):
+ # Create a dict to pass the variables
+ self.deg_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['degree']
+ self.q_norm_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['qnorm']
+ self.coeffs_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['coeffs']
+ self.basis_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['multi_indices']
+ self.score_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['LOOCVScore']
+ self.clf_poly[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['clf_poly']
+ #self.LCerror[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['LCerror']
+
+ if verbose:
+ print(f"\n>>>> Training the {self.meta_model_type} metamodel"
+ " sucessfully completed. <<<<<<\n")
+
+ # -------------------------------------------------------------------------
+ def update_pce_coeffs(self, X, y, out_dict = None):
+ """
+ Updates the PCE coefficents using only the ordinary least square method
+ for the fast version of the bootstrapping.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_params)
+ Training set.
+ y : array of shape (n_samples, n_outs)
+ The (transformed) model responses.
+ out_dict : dict
+ The training output dictionary of the first iteration, i.e.
+ the surrogate model for the original experimental design.
+
+ Returns
+ -------
+ final_out_dict : dict
+ The updated training output dictionary.
+
+ """
+ # Make a copy
+ final_out_dict = copy.deepcopy(out_dict)
+
+ # Loop over the points
+ for i in range(y.shape[1]):
+
+
+ # Extract nonzero basis indices
+ nnz_idx = np.nonzero(out_dict[i]['coeffs'])[0]
+ if len(nnz_idx) != 0:
+ basis_indices = out_dict[i]['multi_indices']
+
+ # Evaluate the multivariate polynomials on CollocationPoints
+ psi = self.create_psi(basis_indices, self.univ_p_val)
+
+ # Calulate the cofficients of surrogate model
+ updated_out = self.regression(
+ psi, y[:, i], basis_indices, reg_method='OLS',
+ sparsity=False
+ )
+
+ # Update coeffs in out_dict
+ final_out_dict[i]['coeffs'][nnz_idx] = updated_out['coeffs']
+
+ return final_out_dict
+
+ # -------------------------------------------------------------------------
+ def add_InputSpace(self):
+ """
+ Instanciates experimental design object.
+
+ Returns
+ -------
+ None.
+
+ """
+ self.InputSpace = InputSpace(self.input_obj,
+ meta_Model_type=self.meta_model_type)
+
+ # -------------------------------------------------------------------------
+ def univ_basis_vals(self, samples, n_max=None):
+ """
+ Evaluates univariate regressors along input directions.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params)
+ Samples.
+ n_max : int, optional
+ Maximum polynomial degree. The default is `None`.
+
+ Returns
+ -------
+ univ_basis: array of shape (n_samples, n_params, n_max+1)
+ All univariate regressors up to n_max.
+ """
+ # Extract information
+ poly_types = self.InputSpace.poly_types
+ if samples.ndim != 2:
+ samples = samples.reshape(1, len(samples))
+ n_max = np.max(self.pce_deg) if n_max is None else n_max
+
+ # Extract poly coeffs
+ if self.InputSpace.input_data_given or self.InputSpace.apce:
+ apolycoeffs = self.polycoeffs
+ else:
+ apolycoeffs = None
+
+ # Evaluate univariate basis
+ univ_basis = eval_univ_basis(samples, n_max, poly_types, apolycoeffs)
+
+ return univ_basis
+
+ # -------------------------------------------------------------------------
+ def create_psi(self, basis_indices, univ_p_val):
+ """
+ This function assemble the design matrix Psi from the given basis index
+ set INDICES and the univariate polynomial evaluations univ_p_val.
+
+ Parameters
+ ----------
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ univ_p_val : array of (n_samples, n_params, n_max+1)
+ All univariate regressors up to `n_max`.
+
+ Raises
+ ------
+ ValueError
+ n_terms in arguments do not match.
+
+ Returns
+ -------
+ psi : array of shape (n_samples, n_terms)
+ Multivariate regressors.
+
+ """
+ # Check if BasisIndices is a sparse matrix
+ sparsity = sp.sparse.issparse(basis_indices)
+ if sparsity:
+ basis_indices = basis_indices.toarray()
+
+ # Initialization and consistency checks
+ # number of input variables
+ n_params = univ_p_val.shape[1]
+
+ # Size of the experimental design
+ n_samples = univ_p_val.shape[0]
+
+ # number of basis terms
+ n_terms = basis_indices.shape[0]
+
+ # check that the variables have consistent sizes
+ if n_params != basis_indices.shape[1]:
+ raise ValueError(
+ f"The shapes of basis_indices ({basis_indices.shape[1]}) and "
+ f"univ_p_val ({n_params}) don't match!!"
+ )
+
+ # Preallocate the Psi matrix for performance
+ psi = np.ones((n_samples, n_terms))
+ # Assemble the Psi matrix
+ for m in range(basis_indices.shape[1]):
+ aa = np.where(basis_indices[:, m] > 0)[0]
+ try:
+ basisIdx = basis_indices[aa, m]
+ bb = univ_p_val[:, m, basisIdx].reshape(psi[:, aa].shape)
+ psi[:, aa] = np.multiply(psi[:, aa], bb)
+ except ValueError as err:
+ raise err
+ return psi
+
+ # -------------------------------------------------------------------------
+ def regression(self, X, y, basis_indices, reg_method=None, sparsity=True):
+ """
+ Fit regression using the regression method provided.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_features)
+ Training vector, where n_samples is the number of samples and
+ n_features is the number of features.
+ y : array of shape (n_samples,)
+ Target values.
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ reg_method : str, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ return_out_dict : Dict
+ Fitted estimator, spareMulti-Index, sparseX and coefficients.
+
+ """
+ if reg_method is None:
+ reg_method = self.pce_reg_method
+
+ bias_term = self.dim_red_method.lower() != 'pca'
+
+ compute_score = True if self.verbose else False
+
+ # inverse of the observed variance of the data
+ if np.var(y) != 0:
+ Lambda = 1 / np.var(y)
+ else:
+ Lambda = 1e-6
+
+ # Bayes sparse adaptive aPCE
+ if reg_method.lower() == 'ols':
+ clf_poly = lm.LinearRegression(fit_intercept=False)
+ elif reg_method.lower() == 'brr':
+ clf_poly = lm.BayesianRidge(n_iter=1000, tol=1e-7,
+ fit_intercept=False,
+ #normalize=True,
+ compute_score=compute_score,
+ alpha_1=1e-04, alpha_2=1e-04,
+ lambda_1=Lambda, lambda_2=Lambda)
+ clf_poly.converged = True
+
+ elif reg_method.lower() == 'ard':
+ if X.shape[0]<2:
+ raise ValueError('Regression with ARD can only be performed for more than 2 samples')
+ clf_poly = lm.ARDRegression(fit_intercept=False,
+ #normalize=True,
+ compute_score=compute_score,
+ n_iter=1000, tol=0.0001,
+ alpha_1=1e-3, alpha_2=1e-3,
+ lambda_1=Lambda, lambda_2=Lambda)
+
+ elif reg_method.lower() == 'fastard':
+ clf_poly = RegressionFastARD(fit_intercept=False,
+ normalize=True,
+ compute_score=compute_score,
+ n_iter=300, tol=1e-10)
+
+ elif reg_method.lower() == 'bcs':
+ if X.shape[0]<10:
+ raise ValueError('Regression with BCS can only be performed for more than 10 samples')
+ clf_poly = RegressionFastLaplace(fit_intercept=False,
+ bias_term=bias_term,
+ n_iter=1000, tol=1e-7)
+
+ elif reg_method.lower() == 'lars':
+ if X.shape[0]<10:
+ raise ValueError('Regression with LARS can only be performed for more than 5 samples')
+ clf_poly = lm.LassoLarsCV(fit_intercept=False)
+
+ elif reg_method.lower() == 'sgdr':
+ clf_poly = lm.SGDRegressor(fit_intercept=False,
+ max_iter=5000, tol=1e-7)
+
+ elif reg_method.lower() == 'omp':
+ clf_poly = OrthogonalMatchingPursuit(fit_intercept=False)
+
+ elif reg_method.lower() == 'vbl':
+ clf_poly = VBLinearRegression(fit_intercept=False)
+
+ elif reg_method.lower() == 'ebl':
+ clf_poly = EBLinearRegression(optimizer='em')
+
+
+ # Training with constraints automatically uses L2
+ if self.apply_constraints:
+ # TODO: set the constraints here
+ # Define the nonlin. constraint
+ nlc = NonlinearConstraint(lambda x: np.matmul(X,x),-1,1.1)
+ self.nlc = nlc
+
+ fun = lambda x: (np.linalg.norm(np.matmul(X, x)-y, ord = 2))**2
+ if self.init_type =='zeros':
+ res = minimize(fun, np.zeros(X.shape[1]), method = 'trust-constr', constraints = self.nlc)
+ if self.init_type == 'nonpi':
+ clf_poly.fit(X, y)
+ coeff = clf_poly.coef_
+ res = minimize(fun, coeff, method = 'trust-constr', constraints = self.nlc)
+
+ coeff = np.array(res.x)
+ clf_poly.coef_ = coeff
+ clf_poly.X = X
+ clf_poly.y = y
+ clf_poly.intercept_ = 0
+
+ # Training without constraints uses chosen regression method
+ else:
+ clf_poly.fit(X, y)
+
+ # Select the nonzero entries of coefficients
+ if sparsity:
+ nnz_idx = np.nonzero(clf_poly.coef_)[0]
+ else:
+ nnz_idx = np.arange(clf_poly.coef_.shape[0])
+
+ # This is for the case where all outputs are zero, thereby
+ # all coefficients are zero
+ if (y == 0).all():
+ nnz_idx = np.insert(np.nonzero(clf_poly.coef_)[0], 0, 0)
+
+ sparse_basis_indices = basis_indices[nnz_idx]
+ sparse_X = X[:, nnz_idx]
+ coeffs = clf_poly.coef_[nnz_idx]
+ clf_poly.coef_ = coeffs
+
+ # Create a dict to pass the outputs
+ return_out_dict = dict()
+ return_out_dict['clf_poly'] = clf_poly
+ return_out_dict['spareMulti-Index'] = sparse_basis_indices
+ return_out_dict['sparePsi'] = sparse_X
+ return_out_dict['coeffs'] = coeffs
+ return return_out_dict
+
+ # -------------------------------------------------------------------------
+ def create_psi(self, basis_indices, univ_p_val):
+ """
+ This function assemble the design matrix Psi from the given basis index
+ set INDICES and the univariate polynomial evaluations univ_p_val.
+
+ Parameters
+ ----------
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ univ_p_val : array of (n_samples, n_params, n_max+1)
+ All univariate regressors up to `n_max`.
+
+ Raises
+ ------
+ ValueError
+ n_terms in arguments do not match.
+
+ Returns
+ -------
+ psi : array of shape (n_samples, n_terms)
+ Multivariate regressors.
+
+ """
+ # Check if BasisIndices is a sparse matrix
+ sparsity = sp.sparse.issparse(basis_indices)
+ if sparsity:
+ basis_indices = basis_indices.toarray()
+
+ # Initialization and consistency checks
+ # number of input variables
+ n_params = univ_p_val.shape[1]
+
+ # Size of the experimental design
+ n_samples = univ_p_val.shape[0]
+
+ # number of basis terms
+ n_terms = basis_indices.shape[0]
+
+ # check that the variables have consistent sizes
+ if n_params != basis_indices.shape[1]:
+ raise ValueError(
+ f"The shapes of basis_indices ({basis_indices.shape[1]}) and "
+ f"univ_p_val ({n_params}) don't match!!"
+ )
+
+ # Preallocate the Psi matrix for performance
+ psi = np.ones((n_samples, n_terms))
+ # Assemble the Psi matrix
+ for m in range(basis_indices.shape[1]):
+ aa = np.where(basis_indices[:, m] > 0)[0]
+ try:
+ basisIdx = basis_indices[aa, m]
+ bb = univ_p_val[:, m, basisIdx].reshape(psi[:, aa].shape)
+ psi[:, aa] = np.multiply(psi[:, aa], bb)
+ except ValueError as err:
+ raise err
+ return psi
+
+ # --------------------------------------------------------------------------------------------------------
+ def adaptive_regression(self, ED_X, ED_Y, varIdx, verbose=False):
+ """
+ Adaptively fits the PCE model by comparing the scores of different
+ degrees and q-norm.
+
+ Parameters
+ ----------
+ ED_X : array of shape (n_samples, n_params)
+ Experimental design.
+ ED_Y : array of shape (n_samples,)
+ Target values, i.e. simulation results for the Experimental design.
+ varIdx : int
+ Index of the output.
+ verbose : bool, optional
+ Print out summary. The default is False.
+
+ Returns
+ -------
+ returnVars : Dict
+ Fitted estimator, best degree, best q-norm, LOOCVScore and
+ coefficients.
+
+ """
+
+ n_samples, n_params = ED_X.shape
+ # Initialization
+ qAllCoeffs, AllCoeffs = {}, {}
+ qAllIndices_Sparse, AllIndices_Sparse = {}, {}
+ qAllclf_poly, Allclf_poly = {}, {}
+ qAllnTerms, AllnTerms = {}, {}
+ qAllLCerror, AllLCerror = {}, {}
+
+ # Extract degree array and qnorm array
+ deg_array = np.array([*self.allBasisIndices], dtype=int)
+ qnorm = [*self.allBasisIndices[str(int(deg_array[0]))]]
+
+ # Some options for EarlyStop
+ errorIncreases = False
+ # Stop degree, if LOO error does not decrease n_checks_degree times
+ n_checks_degree = 3
+ # Stop qNorm, if criterion isn't fulfilled n_checks_qNorm times
+ n_checks_qNorm = 2
+ nqnorms = len(qnorm)
+ qNormEarlyStop = True
+ if nqnorms < n_checks_qNorm+1:
+ qNormEarlyStop = False
+
+ # =====================================================================
+ # basis adaptive polynomial chaos: repeat the calculation by increasing
+ # polynomial degree until the highest accuracy is reached
+ # =====================================================================
+ # For each degree check all q-norms and choose the best one
+ scores = -np.inf * np.ones(deg_array.shape[0])
+ qNormScores = -np.inf * np.ones(nqnorms)
+
+ for degIdx, deg in enumerate(deg_array):
+
+ for qidx, q in enumerate(qnorm):
+
+ # Extract the polynomial basis indices from the pool of
+ # allBasisIndices
+ BasisIndices = self.allBasisIndices[str(deg)][str(q)]
+
+ # Assemble the Psi matrix
+ Psi = self.create_psi(BasisIndices, self.univ_p_val)
+
+ # Calulate the cofficients of the meta model
+ outs = self.regression(Psi, ED_Y, BasisIndices)
+
+ # Calculate and save the score of LOOCV
+ score, LCerror = self.corr_loocv_error(outs['clf_poly'],
+ outs['sparePsi'],
+ outs['coeffs'],
+ ED_Y)
+
+ # Check the convergence of noise for FastARD
+ if self.pce_reg_method == 'FastARD' and \
+ outs['clf_poly'].alpha_ < np.finfo(np.float32).eps:
+ score = -np.inf
+
+ qNormScores[qidx] = score
+ qAllCoeffs[str(qidx+1)] = outs['coeffs']
+ qAllIndices_Sparse[str(qidx+1)] = outs['spareMulti-Index']
+ qAllclf_poly[str(qidx+1)] = outs['clf_poly']
+ qAllnTerms[str(qidx+1)] = BasisIndices.shape[0]
+ qAllLCerror[str(qidx+1)] = LCerror
+
+ # EarlyStop check
+ # if there are at least n_checks_qNorm entries after the
+ # best one, we stop
+ if qNormEarlyStop and \
+ sum(np.isfinite(qNormScores)) > n_checks_qNorm:
+ # If the error has increased the last two iterations, stop!
+ qNormScores_nonInf = qNormScores[np.isfinite(qNormScores)]
+ deltas = np.sign(np.diff(qNormScores_nonInf))
+ if sum(deltas[-n_checks_qNorm+1:]) == 2:
+ # stop the q-norm loop here
+ break
+ if np.var(ED_Y) == 0:
+ break
+
+ # Store the score in the scores list
+ best_q = np.nanargmax(qNormScores)
+ scores[degIdx] = qNormScores[best_q]
+
+ AllCoeffs[str(degIdx+1)] = qAllCoeffs[str(best_q+1)]
+ AllIndices_Sparse[str(degIdx+1)] = qAllIndices_Sparse[str(best_q+1)]
+ Allclf_poly[str(degIdx+1)] = qAllclf_poly[str(best_q+1)]
+ AllnTerms[str(degIdx+1)] = qAllnTerms[str(best_q+1)]
+ AllLCerror[str(degIdx+1)] = qAllLCerror[str(best_q+1)]
+
+ # Check the direction of the error (on average):
+ # if it increases consistently stop the iterations
+ if len(scores[scores != -np.inf]) > n_checks_degree:
+ scores_nonInf = scores[scores != -np.inf]
+ ss = np.sign(scores_nonInf - np.max(scores_nonInf))
+ # ss<0 error decreasing
+ errorIncreases = np.sum(np.sum(ss[-2:])) <= -1*n_checks_degree
+
+ if errorIncreases:
+ break
+
+ # Check only one degree, if target matrix has zero variance
+ if np.var(ED_Y) == 0:
+ break
+
+ # ------------------ Summary of results ------------------
+ # Select the one with the best score and save the necessary outputs
+ best_deg = np.nanargmax(scores)+1
+ coeffs = AllCoeffs[str(best_deg)]
+ basis_indices = AllIndices_Sparse[str(best_deg)]
+ clf_poly = Allclf_poly[str(best_deg)]
+ LOOCVScore = np.nanmax(scores)
+ P = AllnTerms[str(best_deg)]
+ LCerror = AllLCerror[str(best_deg)]
+ degree = deg_array[np.nanargmax(scores)]
+ qnorm = float(qnorm[best_q])
+
+ # ------------------ Print out Summary of results ------------------
+ if self.verbose:
+ # Create PSI_Sparse by removing redundent terms
+ nnz_idx = np.nonzero(coeffs)[0]
+ BasisIndices_Sparse = basis_indices[nnz_idx]
+
+ print(f'Output variable {varIdx+1}:')
+ print('The estimation of PCE coefficients converged at polynomial '
+ f'degree {deg_array[best_deg-1]} with '
+ f'{len(BasisIndices_Sparse)} terms (Sparsity index = '
+ f'{round(len(BasisIndices_Sparse)/P, 3)}).')
+
+ print(f'Final ModLOO error estimate: {1-max(scores):.3e}')
+ print('\n'+'-'*50)
+
+ if verbose:
+ print('='*50)
+ print(' '*10 + ' Summary of results ')
+ print('='*50)
+
+ print("Scores:\n", scores)
+ print("Degree of best score:", self.deg_array[best_deg-1])
+ print("No. of terms:", len(basis_indices))
+ print("Sparsity index:", round(len(basis_indices)/P, 3))
+ print("Best Indices:\n", basis_indices)
+
+ if self.pce_reg_method in ['BRR', 'ARD']:
+ fig, ax = plt.subplots(figsize=(12, 10))
+ plt.title("Marginal log-likelihood")
+ plt.plot(clf_poly.scores_, color='navy', linewidth=2)
+ plt.ylabel("Score")
+ plt.xlabel("Iterations")
+ if self.pce_reg_method.lower() == 'bbr':
+ text = f"$\\alpha={clf_poly.alpha_:.1f}$\n"
+ f"$\\lambda={clf_poly.lambda_:.3f}$\n"
+ f"$L={clf_poly.scores_[-1]:.1f}$"
+ else:
+ text = f"$\\alpha={clf_poly.alpha_:.1f}$\n$"
+ f"\\L={clf_poly.scores_[-1]:.1f}$"
+
+ plt.text(0.75, 0.5, text, fontsize=18, transform=ax.transAxes)
+ plt.show()
+ print('='*80)
+
+ # Create a dict to pass the outputs
+ returnVars = dict()
+ returnVars['clf_poly'] = clf_poly
+ returnVars['degree'] = degree
+ returnVars['qnorm'] = qnorm
+ returnVars['coeffs'] = coeffs
+ returnVars['multi_indices'] = basis_indices
+ returnVars['LOOCVScore'] = LOOCVScore
+ returnVars['LCerror'] = LCerror
+
+ return returnVars
+
+ # -------------------------------------------------------------------------
+ def corr_loocv_error(self, clf, psi, coeffs, y):
+ """
+ Calculates the corrected LOO error for regression on regressor
+ matrix `psi` that generated the coefficients based on [1] and [2].
+
+ [1] Blatman, G., 2009. Adaptive sparse polynomial chaos expansions for
+ uncertainty propagation and sensitivity analysis (Doctoral
+ dissertation, Clermont-Ferrand 2).
+
+ [2] Blatman, G. and Sudret, B., 2011. Adaptive sparse polynomial chaos
+ expansion based on least angle regression. Journal of computational
+ Physics, 230(6), pp.2345-2367.
+
+ Parameters
+ ----------
+ clf : object
+ Fitted estimator.
+ psi : array of shape (n_samples, n_features)
+ The multivariate orthogonal polynomials (regressor).
+ coeffs : array-like of shape (n_features,)
+ Estimated cofficients.
+ y : array of shape (n_samples,)
+ Target values.
+
+ Returns
+ -------
+ R_2 : float
+ LOOCV Validation score (1-LOOCV erro).
+ residual : array of shape (n_samples,)
+ Residual values (y - predicted targets).
+
+ """
+ psi = np.array(psi, dtype=float)
+
+ # Create PSI_Sparse by removing redundent terms
+ nnz_idx = np.nonzero(coeffs)[0]
+ if len(nnz_idx) == 0:
+ nnz_idx = [0]
+ psi_sparse = psi[:, nnz_idx]
+
+ # NrCoeffs of aPCEs
+ P = len(nnz_idx)
+ # NrEvaluation (Size of experimental design)
+ N = psi.shape[0]
+
+ # Build the projection matrix
+ PsiTPsi = np.dot(psi_sparse.T, psi_sparse)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12: #and \
+ # np.linalg.cond(PsiTPsi) < 1/sys.float_info.epsilon:
+ # faster
+ try:
+ M = sp.linalg.solve(PsiTPsi,
+ sp.sparse.eye(PsiTPsi.shape[0]).toarray())
+ except:
+ raise AttributeError('There are too few samples for the corrected loo-cv error. Fit surrogate on at least as many samples as parameters to use this')
+ else:
+ # stabler
+ M = np.linalg.pinv(PsiTPsi)
+
+ # h factor (the full matrix is not calculated explicitly,
+ # only the trace is, to save memory)
+ PsiM = np.dot(psi_sparse, M)
+
+ h = np.sum(np.multiply(PsiM, psi_sparse), axis=1, dtype=np.longdouble)#float128)
+
+ # ------ Calculate Error Loocv for each measurement point ----
+ # Residuals
+ try:
+ residual = clf.predict(psi) - y
+ except:
+ residual = np.dot(psi, coeffs) - y
+
+ # Variance
+ var_y = np.var(y)
+
+ if var_y == 0:
+ norm_emp_error = 0
+ loo_error = 0
+ LCerror = np.zeros((y.shape))
+ return 1-loo_error, LCerror
+ else:
+ norm_emp_error = np.mean(residual**2)/var_y
+
+ # LCerror = np.divide(residual, (1-h))
+ LCerror = residual / (1-h)
+ loo_error = np.mean(np.square(LCerror)) / var_y
+ # if there are NaNs, just return an infinite LOO error (this
+ # happens, e.g., when a strongly underdetermined problem is solved)
+ if np.isnan(loo_error):
+ loo_error = np.inf
+
+ # Corrected Error for over-determined system
+ tr_M = np.trace(M)
+ if tr_M < 0 or abs(tr_M) > 1e6:
+ tr_M = np.trace(np.linalg.pinv(np.dot(psi.T, psi)))
+
+ # Over-determined system of Equation
+ if N > P:
+ T_factor = N/(N-P) * (1 + tr_M)
+
+ # Under-determined system of Equation
+ else:
+ T_factor = np.inf
+
+ corrected_loo_error = loo_error * T_factor
+
+ R_2 = 1 - corrected_loo_error
+
+ return R_2, LCerror
+
+ # -------------------------------------------------------------------------
+ def pca_transformation(self, target, verbose=False):
+ """
+ Transforms the targets (outputs) via Principal Component Analysis
+
+ Parameters
+ ----------
+ target : array of shape (n_samples,)
+ Target values.
+
+ Returns
+ -------
+ pca : obj
+ Fitted sklearnPCA object.
+ OutputMatrix : array of shape (n_samples,)
+ Transformed target values.
+ n_pca_components : int
+ Number of selected principal components.
+
+ """
+ # Transform via Principal Component Analysis
+ if hasattr(self, 'var_pca_threshold'):
+ var_pca_threshold = self.var_pca_threshold
+ else:
+ var_pca_threshold = 100.0
+ n_samples, n_features = target.shape
+
+ if hasattr(self, 'n_pca_components'):
+ n_pca_components = self.n_pca_components
+ else:
+ # Instantiate and fit sklearnPCA object
+ covar_matrix = sklearnPCA(n_components=None)
+ covar_matrix.fit(target)
+ var = np.cumsum(np.round(covar_matrix.explained_variance_ratio_,
+ decimals=5)*100)
+ # Find the number of components to explain self.varPCAThreshold of
+ # variance
+ try:
+ n_components = np.where(var >= var_pca_threshold)[0][0] + 1
+ except IndexError:
+ n_components = min(n_samples, n_features)
+
+ n_pca_components = min(n_samples, n_features, n_components)
+
+ # Print out a report
+ if verbose:
+ print()
+ print('-' * 50)
+ print(f"PCA transformation is performed with {n_pca_components}"
+ " components.")
+ print('-' * 50)
+ print()
+
+ # Fit and transform with the selected number of components
+ pca = sklearnPCA(n_components=n_pca_components, svd_solver='arpack')
+ scaled_target = pca.fit_transform(target)
+
+ return pca, scaled_target, n_pca_components
+
+ # -------------------------------------------------------------------------
+ def gaussian_process_emulator(self, X, y, nug_term=None, autoSelect=False,
+ varIdx=None):
+ """
+ Fits a Gaussian Process Emulator to the target given the training
+ points.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_params)
+ Training points.
+ y : array of shape (n_samples,)
+ Target values.
+ nug_term : float, optional
+ Nugget term. The default is None, i.e. variance of y.
+ autoSelect : bool, optional
+ Loop over some kernels and select the best. The default is False.
+ varIdx : int, optional
+ The index number. The default is None.
+
+ Returns
+ -------
+ gp : object
+ Fitted estimator.
+
+ """
+
+ nug_term = nug_term if nug_term else np.var(y)
+
+ Kernels = [nug_term * kernels.RBF(length_scale=1.0,
+ length_scale_bounds=(1e-25, 1e15)),
+ nug_term * kernels.RationalQuadratic(length_scale=0.2,
+ alpha=1.0),
+ nug_term * kernels.Matern(length_scale=1.0,
+ length_scale_bounds=(1e-15, 1e5),
+ nu=1.5)]
+
+ # Automatic selection of the kernel
+ if autoSelect:
+ gp = {}
+ BME = []
+ for i, kernel in enumerate(Kernels):
+ gp[i] = GaussianProcessRegressor(kernel=kernel,
+ n_restarts_optimizer=3,
+ normalize_y=False)
+
+ # Fit to data using Maximum Likelihood Estimation
+ gp[i].fit(X, y)
+
+ # Store the MLE as BME score
+ BME.append(gp[i].log_marginal_likelihood())
+
+ gp = gp[np.argmax(BME)]
+
+ else:
+ gp = GaussianProcessRegressor(kernel=Kernels[0],
+ n_restarts_optimizer=3,
+ normalize_y=False)
+ gp.fit(X, y)
+
+ # Compute score
+ if varIdx is not None:
+ Score = gp.score(X, y)
+ print('-'*50)
+ print(f'Output variable {varIdx}:')
+ print('The estimation of GPE coefficients converged,')
+ print(f'with the R^2 score: {Score:.3f}')
+ print('-'*50)
+
+ return gp
+
+ # -------------------------------------------------------------------------
+ def eval_metamodel(self, samples):
+ """
+ Evaluates meta-model at the requested samples. One can also generate
+ nsamples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate meta-model at. The default is None.
+ nsamples : int, optional
+ Number of samples to generate, if no `samples` is provided. The
+ default is None.
+ sampling_method : str, optional
+ Type of sampling, if no `samples` is provided. The default is
+ 'random'.
+ return_samples : bool, optional
+ Retun samples, if no `samples` is provided. The default is False.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean of the predictions.
+ std_pred : dict
+ Standard deviatioon of the predictions.
+ """
+ # Transform into np array - can also be given as list
+ samples = np.array(samples)
+
+ # Transform samples to the independent space
+ samples = self.InputSpace.transform(
+ samples,
+ method='user'
+ )
+ # Compute univariate bases for the given samples
+ if self.meta_model_type.lower() != 'gpe':
+ univ_p_val = self.univ_basis_vals(
+ samples,
+ n_max=np.max(self.pce_deg)
+ )
+
+ mean_pred_b = {}
+ std_pred_b = {}
+ # Loop over bootstrap iterations
+ for b_i in range(self.n_bootstrap_itrs):
+
+ # Extract model dictionary
+ if self.meta_model_type.lower() == 'gpe':
+ model_dict = self.gp_poly[f'b_{b_i+1}']
+ else:
+ model_dict = self.coeffs_dict[f'b_{b_i+1}']
+
+ # Loop over outputs
+ mean_pred = {}
+ std_pred = {}
+ for output, values in model_dict.items():
+
+ mean = np.empty((len(samples), len(values)))
+ std = np.empty((len(samples), len(values)))
+ idx = 0
+ for in_key, InIdxValues in values.items():
+
+ # Prediction with GPE
+ if self.meta_model_type.lower() == 'gpe':
+ X_T = self.x_scaler[f'b_{b_i+1}'][output].transform(samples)
+ gp = self.gp_poly[f'b_{b_i+1}'][output][in_key]
+ y_mean, y_std = gp.predict(X_T, return_std=True)
+
+ else:
+ # Prediction with PCE
+ # Assemble Psi matrix
+ basis = self.basis_dict[f'b_{b_i+1}'][output][in_key]
+ psi = self.create_psi(basis, univ_p_val)
+
+ # Prediction
+ if self.bootstrap_method != 'fast' or b_i == 0:
+ # with error bar, i.e. use clf_poly
+ clf_poly = self.clf_poly[f'b_{b_i+1}'][output][in_key]
+ try:
+ y_mean, y_std = clf_poly.predict(
+ psi, return_std=True
+ )
+ except TypeError:
+ y_mean = clf_poly.predict(psi)
+ y_std = np.zeros_like(y_mean)
+ else:
+ # without error bar
+ coeffs = self.coeffs_dict[f'b_{b_i+1}'][output][in_key]
+ y_mean = np.dot(psi, coeffs)
+ y_std = np.zeros_like(y_mean)
+
+ mean[:, idx] = y_mean
+ std[:, idx] = y_std
+ idx += 1
+
+ # Save predictions for each output
+ if self.dim_red_method.lower() == 'pca':
+ PCA = self.pca[f'b_{b_i+1}'][output]
+ mean_pred[output] = PCA.inverse_transform(mean)
+ std_pred[output] = np.zeros(mean.shape)
+ else:
+ mean_pred[output] = mean
+ std_pred[output] = std
+
+ # Save predictions for each bootstrap iteration
+ mean_pred_b[b_i] = mean_pred
+ std_pred_b[b_i] = std_pred
+
+ # Change the order of nesting
+ mean_pred_all = {}
+ for i in sorted(mean_pred_b):
+ for k, v in mean_pred_b[i].items():
+ if k not in mean_pred_all:
+ mean_pred_all[k] = [None] * len(mean_pred_b)
+ mean_pred_all[k][i] = v
+
+ # Compute the moments of predictions over the predictions
+ for output in self.out_names:
+ # Only use bootstraps with finite values
+ finite_rows = np.isfinite(
+ mean_pred_all[output]).all(axis=2).all(axis=1)
+ outs = np.asarray(mean_pred_all[output])[finite_rows]
+ # Compute mean
+ mean_pred[output] = np.mean(outs, axis=0)
+ # Compute standard deviation
+ if self.n_bootstrap_itrs > 1:
+ std_pred[output] = np.std(outs, axis=0)
+ else:
+ std_pred[output] = std_pred_b[b_i][output]
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ def create_model_error(self, X, y, Model, name='Calib'):
+ """
+ Fits a GPE-based model error.
+
+ Parameters
+ ----------
+ X : array of shape (n_outputs, n_inputs)
+ Input array. It can contain any forcing inputs or coordinates of
+ extracted data.
+ y : array of shape (n_outputs,)
+ The model response for the MAP parameter set.
+ name : str, optional
+ Calibration or validation. The default is `'Calib'`.
+
+ Returns
+ -------
+ self: object
+ Self object.
+
+ """
+ outputNames = self.out_names
+ self.errorRegMethod = 'GPE'
+ self.errorclf_poly = self.auto_vivification()
+ self.errorScale = self.auto_vivification()
+
+ # Read data
+ # TODO: do this call outside the metamodel
+ MeasuredData = Model.read_observation(case=name)
+
+ # Fitting GPR based bias model
+ for out in outputNames:
+ nan_idx = ~np.isnan(MeasuredData[out])
+ # Select data
+ try:
+ data = MeasuredData[out].values[nan_idx]
+ except AttributeError:
+ data = MeasuredData[out][nan_idx]
+
+ # Prepare the input matrix
+ scaler = MinMaxScaler()
+ delta = data # - y[out][0]
+ BiasInputs = np.hstack((X[out], y[out].reshape(-1, 1)))
+ X_S = scaler.fit_transform(BiasInputs)
+ gp = self.gaussian_process_emulator(X_S, delta)
+
+ self.errorScale[out]["y_1"] = scaler
+ self.errorclf_poly[out]["y_1"] = gp
+
+ return self
+
+ # -------------------------------------------------------------------------
+ def eval_model_error(self, X, y_pred):
+ """
+ Evaluates the error model.
+
+ Parameters
+ ----------
+ X : array
+ Inputs.
+ y_pred : dict
+ Predictions.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean predition of the GPE-based error model.
+ std_pred : dict
+ standard deviation of the GPE-based error model.
+
+ """
+ mean_pred = {}
+ std_pred = {}
+
+ for Outkey, ValuesDict in self.errorclf_poly.items():
+
+ pred_mean = np.zeros_like(y_pred[Outkey])
+ pred_std = np.zeros_like(y_pred[Outkey])
+
+ for Inkey, InIdxValues in ValuesDict.items():
+
+ gp = self.errorclf_poly[Outkey][Inkey]
+ scaler = self.errorScale[Outkey][Inkey]
+
+ # Transform Samples using scaler
+ for j, pred in enumerate(y_pred[Outkey]):
+ BiasInputs = np.hstack((X[Outkey], pred.reshape(-1, 1)))
+ Samples_S = scaler.transform(BiasInputs)
+ y_hat, y_std = gp.predict(Samples_S, return_std=True)
+ pred_mean[j] = y_hat
+ pred_std[j] = y_std
+ # pred_mean[j] += pred
+
+ mean_pred[Outkey] = pred_mean
+ std_pred[Outkey] = pred_std
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ class auto_vivification(dict):
+ """
+ Implementation of perl's AutoVivification feature.
+
+ Source: https://stackoverflow.com/a/651879/18082457
+ """
+
+ def __getitem__(self, item):
+ try:
+ return dict.__getitem__(self, item)
+ except KeyError:
+ value = self[item] = type(self)()
+ return value
+
+ # -------------------------------------------------------------------------
+ def copy_meta_model_opts(self):
+ """
+ This method is a convinient function to copy the metamodel options.
+
+ Returns
+ -------
+ new_MetaModelOpts : object
+ The copied object.
+
+ """
+ # TODO: what properties should be moved to the new object?
+ new_MetaModelOpts = copy.deepcopy(self)
+ new_MetaModelOpts.input_obj = self.input_obj#InputObj
+ new_MetaModelOpts.InputSpace = self.InputSpace
+ #new_MetaModelOpts.InputSpace.meta_Model = 'aPCE'
+ #new_MetaModelOpts.InputSpace.InputObj = self.input_obj
+ #new_MetaModelOpts.InputSpace.ndim = len(self.input_obj.Marginals)
+ new_MetaModelOpts.n_params = len(self.input_obj.Marginals)
+ #new_MetaModelOpts.InputSpace.hdf5_file = None
+
+ return new_MetaModelOpts
+
+ # -------------------------------------------------------------------------
+ def __select_degree(self, ndim, n_samples):
+ """
+ Selects degree based on the number of samples and parameters in the
+ sequential design.
+
+ Parameters
+ ----------
+ ndim : int
+ Dimension of the parameter space.
+ n_samples : int
+ Number of samples.
+
+ Returns
+ -------
+ deg_array: array
+ Array containing the arrays.
+
+ """
+ # Define the deg_array
+ max_deg = np.max(self.pce_deg)
+ min_Deg = np.min(self.pce_deg)
+
+ # TODO: remove the options for sequential?
+ #nitr = n_samples - self.InputSpace.n_init_samples
+
+ # Check q-norm
+ if not np.isscalar(self.pce_q_norm):
+ self.pce_q_norm = np.array(self.pce_q_norm)
+ else:
+ self.pce_q_norm = np.array([self.pce_q_norm])
+
+ def M_uptoMax(maxDeg):
+ n_combo = np.zeros(maxDeg)
+ for i, d in enumerate(range(1, maxDeg+1)):
+ n_combo[i] = math.factorial(ndim+d)
+ n_combo[i] /= math.factorial(ndim) * math.factorial(d)
+ return n_combo
+
+ deg_new = max_deg
+ #d = nitr if nitr != 0 and self.n_params > 5 else 1
+ # d = 1
+ # min_index = np.argmin(abs(M_uptoMax(max_deg)-ndim*n_samples*d))
+ # deg_new = range(1, max_deg+1)[min_index]
+
+ if deg_new > min_Deg and self.pce_reg_method.lower() != 'fastard':
+ deg_array = np.arange(min_Deg, deg_new+1)
+ else:
+ deg_array = np.array([deg_new])
+
+ return deg_array
+
+ def generate_polynomials(self, max_deg=None):
+ # Check for InputSpace
+ if not hasattr(self, 'InputSpace'):
+ raise AttributeError('Generate or add InputSpace before generating polynomials')
+
+ ndim = self.InputSpace.ndim
+ # Create orthogonal polynomial coefficients if necessary
+ if (self.meta_model_type.lower()!='gpe') and max_deg is not None:# and self.input_obj.poly_coeffs_flag:
+ self.polycoeffs = {}
+ for parIdx in tqdm(range(ndim), ascii=True,
+ desc="Computing orth. polynomial coeffs"):
+ poly_coeffs = apoly_construction(
+ self.InputSpace.raw_data[parIdx],
+ max_deg
+ )
+ self.polycoeffs[f'p_{parIdx+1}'] = poly_coeffs
+ else:
+ raise AttributeError('MetaModel cannot generate polynomials in the given scenario!')
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+
+ # Check if its truly a pce-surrogate
+ if self.meta_model_type.lower() == 'gpe':
+ raise AttributeError('Moments can only be computed for pce-type surrogates')
+
+ outputs = self.out_names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(self.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = self.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = self.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = self.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if self.dim_red_method.lower() == 'pca':
+ PCA = self.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = PCA.inverse_transform(np.sqrt(pce_var))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/examples/analytical-function/example_analytical_function.py b/examples/analytical-function/example_analytical_function.py
index 52e7731b576c42cdd29a705865f0a0389f812654..9ec23f8cdaae003c33bac4cfe75e9a1a4d82bc42 100644
--- a/examples/analytical-function/example_analytical_function.py
+++ b/examples/analytical-function/example_analytical_function.py
@@ -141,7 +141,7 @@ if __name__ == "__main__":
# One-shot (normal) or Sequential Adaptive (sequential) Design
ExpDesign.method = 'sequential'
- ExpDesign.n_init_samples = 100#3*ndim
+ ExpDesign.n_init_samples = 140#00#3*ndim
# Sampling methods
# 1) random 2) latin_hypercube 3) sobol 4) halton 5) hammersley
@@ -271,7 +271,7 @@ if __name__ == "__main__":
# BayesOpts.bootstrap_noise = 100
# Bayesian cross validation
- # BayesOpts.bayes_loocv = True
+ BayesOpts.bayes_loocv = True # TODO: test what this does
# Select the inference method
import emcee
diff --git a/examples/model-comparison/bayesvalidrox/__init__.py b/examples/model-comparison/bayesvalidrox/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8e865af80652b8dd29203c2c85f8d1c717e335bc
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/__init__.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+__version__ = "0.0.5"
+
+from .pylink.pylink import PyLinkForwardModel
+from .surrogate_models.surrogate_models import MetaModel
+#from .surrogate_models.meta_model_engine import MetaModelEngine
+from .surrogate_models.engine import Engine
+from .surrogate_models.inputs import Input
+from .post_processing.post_processing import PostProcessing
+from .bayes_inference.bayes_inference import BayesInference
+from .bayes_inference.bayes_model_comparison import BayesModelComparison
+from .bayes_inference.discrepancy import Discrepancy
+
+__all__ = [
+ "__version__",
+ "PyLinkForwardModel",
+ "Input",
+ "Discrepancy",
+ "MetaModel",
+ #"MetaModelEngine",
+ "Engine",
+ "PostProcessing",
+ "BayesInference",
+ "BayesModelComparison"
+ ]
diff --git a/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..beaab3c798a63fcfbc361982388fdf10830a787e
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b1f65d567bf46826c3f2f76b1c730caf924b8f43
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d5ebaedb4c8b77b5d7dbd0a6945f09079d8b10e4
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__init__.py b/examples/model-comparison/bayesvalidrox/bayes_inference/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..df8d935680f96ab487cf087866e8bfd504762945
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayes_inference/__init__.py
@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+from .bayes_inference import BayesInference
+from .mcmc import MCMC
+
+__all__ = [
+ "BayesInference",
+ "MCMC"
+ ]
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..430b9885a8c8bd658da24bbc4ac1a6a0a74f69e6
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..72c63a98588c54dfec12536a99537cfa3a67e0cf
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..287257c1ca9b3f3a6d7e176e006ad432f8c685bf
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e89dfb5e6b3a873ac2f40dcc2084aa52caaedcd6
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..88adfbcabb95e11d86659db42240c5fa87390c2e
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a37330680c29bbfdba5a1bfd98041dda24957604
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_inference.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..57322b839ff7d50ea32d3b36ce011c09cf91e232
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f8cf9ea94d01b774b2d634e4e0caa06ef9bf6843
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b58f792ac36504c702205afc34228600f9bbba77
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/bayes_model_comparison.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bc71688a4cae4f74ec3a67838fca659881c520c9
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5301d0c093a62cd323cbd18cdf99d5eed93ad76c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f154800917aeaef9c499abf13d47fcca6ffc639b
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/discrepancy.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d2122246d54ec5697803cbce28900e077b2306a
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6177da43e7573677803782f2472399b0cb79a672
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b9fe4689524895c2149048d489b22b08e85026df
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/bayes_inference/__pycache__/mcmc.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_inference.py b/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_inference.py
new file mode 100644
index 0000000000000000000000000000000000000000..1898a8ae619597d92bc355ac4249f57019f0aed7
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_inference.py
@@ -0,0 +1,1532 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import os
+import copy
+import pandas as pd
+from tqdm import tqdm
+from scipy import stats
+import scipy.linalg as spla
+import joblib
+import seaborn as sns
+import corner
+import h5py
+import multiprocessing
+import gc
+from sklearn.metrics import mean_squared_error, r2_score
+from sklearn import preprocessing
+from matplotlib.patches import Patch
+import matplotlib.lines as mlines
+from matplotlib.backends.backend_pdf import PdfPages
+import matplotlib.pylab as plt
+
+from .mcmc import MCMC
+
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class BayesInference:
+ """
+ A class to perform Bayesian Analysis.
+
+
+ Attributes
+ ----------
+ MetaModel : obj
+ Meta model object.
+ discrepancy : obj
+ The discrepancy object for the sigma2s, i.e. the diagonal entries
+ of the variance matrix for a multivariate normal likelihood.
+ name : str, optional
+ The type of analysis, either calibration (`Calib`) or validation
+ (`Valid`). The default is `'Calib'`.
+ emulator : bool, optional
+ Analysis with emulator (MetaModel). The default is `True`.
+ bootstrap : bool, optional
+ Bootstrap the analysis. The default is `False`.
+ req_outputs : list, optional
+ The list of requested output to be used for the analysis.
+ The default is `None`. If None, all the defined outputs for the model
+ object is used.
+ selected_indices : dict, optional
+ A dictionary with the selected indices of each model output. The
+ default is `None`. If `None`, all measurement points are used in the
+ analysis.
+ samples : array of shape (n_samples, n_params), optional
+ The samples to be used in the analysis. The default is `None`. If
+ None the samples are drawn from the probablistic input parameter
+ object of the MetaModel object.
+ n_samples : int, optional
+ Number of samples to be used in the analysis. The default is `500000`.
+ If samples is not `None`, this argument will be assigned based on the
+ number of samples given.
+ measured_data : dict, optional
+ A dictionary containing the observation data. The default is `None`.
+ if `None`, the observation defined in the Model object of the
+ MetaModel is used.
+ inference_method : str, optional
+ A method for approximating the posterior distribution in the Bayesian
+ inference step. The default is `'rejection'`, which stands for
+ rejection sampling. A Markov Chain Monte Carlo sampler can be simply
+ selected by passing `'MCMC'`.
+ mcmc_params : dict, optional
+ A dictionary with args required for the Bayesian inference with
+ `MCMC`. The default is `None`.
+
+ Pass the mcmc_params like the following:
+
+ >>> mcmc_params:{
+ 'init_samples': None, # initial samples
+ 'n_walkers': 100, # number of walkers (chain)
+ 'n_steps': 100000, # number of maximum steps
+ 'n_burn': 200, # number of burn-in steps
+ 'moves': None, # Moves for the emcee sampler
+ 'multiprocessing': False, # multiprocessing
+ 'verbose': False # verbosity
+ }
+ The items shown above are the default values. If any parmeter is
+ not defined, the default value will be assigned to it.
+ bayes_loocv : bool, optional
+ Bayesian Leave-one-out Cross Validation. The default is `False`. If
+ `True`, the LOOCV procedure is used to estimate the bayesian Model
+ Evidence (BME).
+ n_bootstrap_itrs : int, optional
+ Number of bootstrap iteration. The default is `1`. If bayes_loocv is
+ `True`, this is qualt to the total length of the observation data
+ set.
+ perturbed_data : array of shape (n_bootstrap_itrs, n_obs), optional
+ User defined perturbed data. The default is `[]`.
+ bootstrap_noise : float, optional
+ A noise level to perturb the data set. The default is `0.05`.
+ just_analysis : bool, optional
+ Justifiability analysis. The default is False.
+ valid_metrics : list, optional
+ List of the validation metrics. The following metrics are supported:
+
+ 1. log_BME : logarithm of the Bayesian model evidence
+ 2. KLD : Kullback-Leibler Divergence
+ 3. inf_entropy: Information entropy
+ The default is `['log_BME']`.
+ plot_post_pred : bool, optional
+ Plot posterior predictive plots. The default is `True`.
+ plot_map_pred : bool, optional
+ Plot the model outputs vs the metamodel predictions for the maximum
+ a posteriori (defined as `max_a_posteriori`) parameter set. The
+ default is `False`.
+ max_a_posteriori : str, optional
+ Maximum a posteriori. `'mean'` and `'mode'` are available. The default
+ is `'mean'`.
+ corner_title_fmt : str, optional
+ Title format for the posterior distribution plot with python
+ package `corner`. The default is `'.2e'`.
+
+ """
+
+ def __init__(self, engine, MetaModel = None, discrepancy=None, emulator=True,
+ name='Calib', bootstrap=False, req_outputs=None,
+ selected_indices=None, samples=None, n_samples=100000,
+ measured_data=None, inference_method='rejection',
+ mcmc_params=None, bayes_loocv=False, n_bootstrap_itrs=1,
+ perturbed_data=[], bootstrap_noise=0.05, just_analysis=False,
+ valid_metrics=['BME'], plot_post_pred=True,
+ plot_map_pred=False, max_a_posteriori='mean',
+ corner_title_fmt='.2e'):
+
+ self.engine = engine
+ self.MetaModel = engine.MetaModel
+ self.Discrepancy = discrepancy
+ self.emulator = emulator
+ self.name = name
+ self.bootstrap = bootstrap
+ self.req_outputs = req_outputs
+ self.selected_indices = selected_indices
+ self.samples = samples
+ self.n_samples = n_samples
+ self.measured_data = measured_data
+ self.inference_method = inference_method
+ self.mcmc_params = mcmc_params
+ self.perturbed_data = perturbed_data
+ self.bayes_loocv = bayes_loocv
+ self.n_bootstrap_itrs = n_bootstrap_itrs
+ self.bootstrap_noise = bootstrap_noise
+ self.just_analysis = just_analysis
+ self.valid_metrics = valid_metrics
+ self.plot_post_pred = plot_post_pred
+ self.plot_map_pred = plot_map_pred
+ self.max_a_posteriori = max_a_posteriori
+ self.corner_title_fmt = corner_title_fmt
+
+ # -------------------------------------------------------------------------
+ def create_inference(self):
+ """
+ Starts the inference.
+
+ Returns
+ -------
+ BayesInference : obj
+ The Bayes inference object.
+
+ """
+
+ # Set some variables
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+ n_params = MetaModel.n_params
+ output_names = Model.Output.names
+ par_names = self.engine.ExpDesign.par_names
+
+ # If the prior is set by the user, take it.
+ if self.samples is None:
+ self.samples = self.engine.ExpDesign.generate_samples(
+ self.n_samples, 'random')
+ else:
+ try:
+ samples = self.samples.values
+ except AttributeError:
+ samples = self.samples
+
+ # Take care of an additional Sigma2s
+ self.samples = samples[:, :n_params]
+
+ # Update number of samples
+ self.n_samples = self.samples.shape[0]
+
+ # ---------- Preparation of observation data ----------
+ # Read observation data and perturb it if requested.
+ if self.measured_data is None:
+ self.measured_data = Model.read_observation(case=self.name)
+ # Convert measured_data to a data frame
+ if not isinstance(self.measured_data, pd.DataFrame):
+ self.measured_data = pd.DataFrame(self.measured_data)
+
+ # Extract the total number of measurement points
+ if self.name.lower() == 'calib':
+ self.n_tot_measurement = Model.n_obs
+ else:
+ self.n_tot_measurement = Model.n_obs_valid
+
+ # Find measurement error (if not given) for post predictive plot
+ if not hasattr(self, 'measurement_error'):
+ if isinstance(self.Discrepancy, dict):
+ Disc = self.Discrepancy['known']
+ else:
+ Disc = self.Discrepancy
+ if isinstance(Disc.parameters, dict):
+ self.measurement_error = {k: np.sqrt(Disc.parameters[k]) for k
+ in Disc.parameters.keys()}
+ else:
+ try:
+ self.measurement_error = np.sqrt(Disc.parameters)
+ except TypeError:
+ pass
+
+ # ---------- Preparation of variance for covariance matrix ----------
+ # Independent and identically distributed
+ total_sigma2 = dict()
+ opt_sigma_flag = isinstance(self.Discrepancy, dict)
+ opt_sigma = None
+ for key_idx, key in enumerate(output_names):
+
+ # Find opt_sigma
+ if opt_sigma_flag and opt_sigma is None:
+ # Option A: known error with unknown bias term
+ opt_sigma = 'A'
+ known_discrepancy = self.Discrepancy['known']
+ self.Discrepancy = self.Discrepancy['infer']
+ sigma2 = np.array(known_discrepancy.parameters[key])
+
+ elif opt_sigma == 'A' or self.Discrepancy.parameters is not None:
+ # Option B: The sigma2 is known (no bias term)
+ if opt_sigma == 'A':
+ sigma2 = np.array(known_discrepancy.parameters[key])
+ else:
+ opt_sigma = 'B'
+ sigma2 = np.array(self.Discrepancy.parameters[key])
+
+ elif not isinstance(self.Discrepancy.InputDisc, str):
+ # Option C: The sigma2 is unknown (bias term including error)
+ opt_sigma = 'C'
+ self.Discrepancy.opt_sigma = opt_sigma
+ n_measurement = self.measured_data[key].values.shape
+ sigma2 = np.zeros((n_measurement[0]))
+
+ total_sigma2[key] = sigma2
+
+ self.Discrepancy.opt_sigma = opt_sigma
+ self.Discrepancy.total_sigma2 = total_sigma2
+
+ # If inferred sigma2s obtained from e.g. calibration are given
+ try:
+ self.sigma2s = self.Discrepancy.get_sample(self.n_samples)
+ except:
+ pass
+
+ # ---------------- Bootstrap & TOM --------------------
+ if self.bootstrap or self.bayes_loocv or self.just_analysis:
+ if len(self.perturbed_data) == 0:
+ # zero mean noise Adding some noise to the observation function
+ self.perturbed_data = self._perturb_data(
+ self.measured_data, output_names
+ )
+ else:
+ self.n_bootstrap_itrs = len(self.perturbed_data)
+
+ # -------- Model Discrepancy -----------
+ if hasattr(self, 'error_model') and self.error_model \
+ and self.name.lower() != 'calib':
+ # Select posterior mean as MAP
+ MAP_theta = self.samples.mean(axis=0).reshape((1, n_params))
+ # MAP_theta = stats.mode(self.samples,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=MAP_theta)
+
+ # Train a GPR meta-model using MAP
+ self.error_MetaModel = MetaModel.create_model_error(
+ self.bias_inputs, y_MAP, Name=self.name
+ )
+
+ # -----------------------------------------------------
+ # ----- Loop over the perturbed observation data ------
+ # -----------------------------------------------------
+ # Initilize arrays
+ logLikelihoods = np.zeros((self.n_samples, self.n_bootstrap_itrs),
+ dtype=np.float16)
+ BME_Corr = np.zeros((self.n_bootstrap_itrs))
+ log_BME = np.zeros((self.n_bootstrap_itrs))
+ KLD = np.zeros((self.n_bootstrap_itrs))
+ inf_entropy = np.zeros((self.n_bootstrap_itrs))
+
+ # Compute the prior predtions
+ # Evaluate the MetaModel
+ if self.emulator:
+ y_hat, y_std = MetaModel.eval_metamodel(samples=self.samples)
+ self.__mean_pce_prior_pred = y_hat
+ self._std_pce_prior_pred = y_std
+
+ # Correct the predictions with Model discrepancy
+ if hasattr(self, 'error_model') and self.error_model:
+ y_hat_corr, y_std = self.error_MetaModel.eval_model_error(
+ self.bias_inputs, self.__mean_pce_prior_pred
+ )
+ self.__mean_pce_prior_pred = y_hat_corr
+ self._std_pce_prior_pred = y_std
+
+ # Surrogate model's error using RMSE of test data
+ if hasattr(MetaModel, 'rmse'):
+ surrError = MetaModel.rmse
+ else:
+ surrError = None
+
+ else:
+ # Evaluate the original model
+ self.__model_prior_pred = self._eval_model(
+ samples=self.samples, key='PriorPred'
+ )
+ surrError = None
+
+ # Start the likelihood-BME computations for the perturbed data
+ for itr_idx, data in tqdm(
+ enumerate(self.perturbed_data),
+ total=self.n_bootstrap_itrs,
+ desc="Bootstrapping the BME calculations", ascii=True
+ ):
+
+ # ---------------- Likelihood calculation ----------------
+ if self.emulator:
+ model_evals = self.__mean_pce_prior_pred
+ else:
+ model_evals = self.__model_prior_pred
+
+ # Leave one out
+ if self.bayes_loocv or self.just_analysis:
+ self.selected_indices = np.nonzero(data)[0]
+
+ # Prepare data dataframe
+ nobs = list(self.measured_data.count().values[1:])
+ numbers = list(np.cumsum(nobs))
+ indices = list(zip([0] + numbers, numbers))
+ data_dict = {
+ output_names[i]: data[j:k] for i, (j, k) in
+ enumerate(indices)
+ }
+ #print(output_names)
+ #print(indices)
+ #print(numbers)
+ #print(nobs)
+ #print(self.measured_data)
+ #for i, (j, k) in enumerate(indices):
+ # print(i,j,k)
+ #print(data)
+ #print(data_dict)
+ #stop
+
+ # Unknown sigma2
+ if opt_sigma == 'C' or hasattr(self, 'sigma2s'):
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ sigma2=self.sigma2s, std=surrError
+ )
+ else:
+ # known sigma2
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ std=surrError
+ )
+
+ # ---------------- BME Calculations ----------------
+ # BME (log)
+ log_BME[itr_idx] = np.log(
+ np.nanmean(np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.longdouble))#float128))
+ )
+
+ # BME correction when using Emulator
+ if self.emulator:
+ BME_Corr[itr_idx] = self.__corr_factor_BME(
+ data_dict, total_sigma2, log_BME[itr_idx]
+ )
+
+ # Rejection Step
+ if 'kld' in list(map(str.lower, self.valid_metrics)) and\
+ 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, self.n_samples)[0]
+
+ # Reject the poorly performed prior
+ Likelihoods = np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.float64)
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ posterior = self.samples[accepted]
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(
+ logLikelihoods[:, itr_idx][accepted]
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ KLD[itr_idx] = postExpLikelihoods - log_BME[itr_idx]
+
+ # Posterior-based expectation of prior densities
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ n_thread = int(0.875 * multiprocessing.cpu_count())
+ with multiprocessing.Pool(n_thread) as p:
+ postExpPrior = np.mean(np.concatenate(
+ p.map(
+ self.engine.ExpDesign.JDist.pdf,
+ np.array_split(posterior.T, n_thread, axis=1))
+ )
+ )
+ # Information Entropy based on Entropy paper Eq. 38
+ inf_entropy[itr_idx] = log_BME[itr_idx] - postExpPrior - \
+ postExpLikelihoods
+
+ # Clear memory
+ gc.collect(generation=2)
+
+ # ---------- Store metrics for perturbed data set ----------------
+ # Likelihoods (Size: n_samples, n_bootstrap_itr)
+ self.log_likes = logLikelihoods
+
+ # BME (log), KLD, infEntropy (Size: 1,n_bootstrap_itr)
+ self.log_BME = log_BME
+
+ # BMECorrFactor (log) (Size: 1,n_bootstrap_itr)
+ if self.emulator:
+ self.log_BME_corr_factor = BME_Corr
+
+ if 'kld' in list(map(str.lower, self.valid_metrics)):
+ self.KLD = KLD
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ self.inf_entropy = inf_entropy
+
+ # BME = BME + BMECorrFactor
+ if self.emulator:
+ self.log_BME += self.log_BME_corr_factor
+
+ # ---------------- Parameter Bayesian inference ----------------
+ if self.inference_method.lower() == 'mcmc':
+ # Instantiate the MCMC object
+ MCMC_Obj = MCMC(self)
+ self.posterior_df = MCMC_Obj.run_sampler(
+ self.measured_data, total_sigma2
+ )
+
+ elif self.name.lower() == 'valid':
+ # Convert to a dataframe if samples are provided after calibration.
+ self.posterior_df = pd.DataFrame(self.samples, columns=par_names)
+
+ else:
+ # Rejection sampling
+ self.posterior_df = self._rejection_sampling()
+
+ # Provide posterior's summary
+ print('\n')
+ print('-'*15 + 'Posterior summary' + '-'*15)
+ pd.options.display.max_columns = None
+ pd.options.display.max_rows = None
+ print(self.posterior_df.describe())
+ print('-'*50)
+
+ # -------- Model Discrepancy -----------
+ if hasattr(self, 'error_model') and self.error_model \
+ and self.name.lower() == 'calib':
+ if self.inference_method.lower() == 'mcmc':
+ self.error_MetaModel = MCMC_Obj.error_MetaModel
+ else:
+ # Select posterior mean as MAP
+ if opt_sigma == "B":
+ posterior_df = self.posterior_df.values
+ else:
+ posterior_df = self.posterior_df.values[:, :-Model.n_outputs]
+
+ # Select posterior mean as Maximum a posteriori
+ map_theta = posterior_df.mean(axis=0).reshape((1, n_params))
+ # map_theta = stats.mode(Posterior_df,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=map_theta)
+
+ # Train a GPR meta-model using MAP
+ self.error_MetaModel = MetaModel.create_model_error(
+ self.bias_inputs, y_MAP, Name=self.name
+ )
+
+ # -------- Posterior perdictive -----------
+ self._posterior_predictive()
+
+ # -----------------------------------------------------
+ # ------------------ Visualization --------------------
+ # -----------------------------------------------------
+ # Create Output directory, if it doesn't exist already.
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ os.makedirs(out_dir, exist_ok=True)
+
+ # -------- Posteior parameters --------
+ if opt_sigma != "B":
+ par_names.extend(
+ [self.Discrepancy.InputDisc.Marginals[i].name for i
+ in range(len(self.Discrepancy.InputDisc.Marginals))]
+ )
+ # Pot with corner
+ figPosterior = corner.corner(self.posterior_df.to_numpy(),
+ labels=par_names,
+ quantiles=[0.15, 0.5, 0.85],
+ show_titles=True,
+ title_fmt=self.corner_title_fmt,
+ labelpad=0.2,
+ use_math_text=True,
+ title_kwargs={"fontsize": 28},
+ plot_datapoints=False,
+ plot_density=False,
+ fill_contours=True,
+ smooth=0.5,
+ smooth1d=0.5)
+
+ # Loop over axes and set x limits
+ if opt_sigma == "B":
+ axes = np.array(figPosterior.axes).reshape(
+ (len(par_names), len(par_names))
+ )
+ for yi in range(len(par_names)):
+ ax = axes[yi, yi]
+ ax.set_xlim(self.engine.ExpDesign.bound_tuples[yi])
+ for xi in range(yi):
+ ax = axes[yi, xi]
+ ax.set_xlim(self.engine.ExpDesign.bound_tuples[xi])
+ plt.close()
+
+ # Turn off gridlines
+ for ax in figPosterior.axes:
+ ax.grid(False)
+
+ if self.emulator:
+ plotname = f'/Posterior_Dist_{Model.name}_emulator'
+ else:
+ plotname = f'/Posterior_Dist_{Model.name}'
+
+ figPosterior.set_size_inches((24, 16))
+ figPosterior.savefig(f'./{out_dir}{plotname}.pdf',
+ bbox_inches='tight')
+
+ # -------- Plot MAP --------
+ if self.plot_map_pred:
+ self._plot_max_a_posteriori()
+
+ # -------- Plot log_BME dist --------
+ if self.bootstrap:
+
+ # Computing the TOM performance
+ self.log_BME_tom = stats.chi2.rvs(
+ self.n_tot_measurement, size=self.log_BME.shape[0]
+ )
+
+ fig, ax = plt.subplots()
+ sns.kdeplot(self.log_BME_tom, ax=ax, color="green", shade=True)
+ sns.kdeplot(
+ self.log_BME, ax=ax, color="blue", shade=True,
+ label='Model BME')
+
+ ax.set_xlabel('log$_{10}$(BME)')
+ ax.set_ylabel('Probability density')
+
+ legend_elements = [
+ Patch(facecolor='green', edgecolor='green', label='TOM BME'),
+ Patch(facecolor='blue', edgecolor='blue', label='Model BME')
+ ]
+ ax.legend(handles=legend_elements)
+
+ if self.emulator:
+ plotname = f'/BME_hist_{Model.name}_emulator'
+ else:
+ plotname = f'/BME_hist_{Model.name}'
+
+ plt.savefig(f'./{out_dir}{plotname}.pdf', bbox_inches='tight')
+ plt.show()
+ plt.close()
+
+ # -------- Posteior perdictives --------
+ if self.plot_post_pred:
+ # Plot the posterior predictive
+ self._plot_post_predictive()
+
+ return self
+
+ # -------------------------------------------------------------------------
+ def _perturb_data(self, data, output_names):
+ """
+ Returns an array with n_bootstrap_itrs rowsof perturbed data.
+ The first row includes the original observation data.
+ If `self.bayes_loocv` is True, a 2d-array will be returned with
+ repeated rows and zero diagonal entries.
+
+ Parameters
+ ----------
+ data : pandas DataFrame
+ Observation data.
+ output_names : list
+ List of the output names.
+
+ Returns
+ -------
+ final_data : array
+ Perturbed data set.
+
+ """
+ noise_level = self.bootstrap_noise
+ obs_data = data[output_names].values
+ n_measurement, n_outs = obs_data.shape
+ self.n_tot_measurement = obs_data[~np.isnan(obs_data)].shape[0]
+ # Number of bootstrap iterations
+ if self.bayes_loocv:
+ self.n_bootstrap_itrs = self.n_tot_measurement
+
+ # Pass loocv dataset
+ if self.bayes_loocv:
+ obs = obs_data.T[~np.isnan(obs_data.T)]
+ final_data = np.repeat(np.atleast_2d(obs), self.n_bootstrap_itrs,
+ axis=0)
+ np.fill_diagonal(final_data, 0)
+ return final_data
+
+ else:
+ final_data = np.zeros(
+ (self.n_bootstrap_itrs, self.n_tot_measurement)
+ )
+ final_data[0] = obs_data.T[~np.isnan(obs_data.T)]
+ for itrIdx in range(1, self.n_bootstrap_itrs):
+ data = np.zeros((n_measurement, n_outs))
+ for idx in range(len(output_names)):
+ std = np.nanstd(obs_data[:, idx])
+ if std == 0:
+ std = 0.001
+ noise = std * noise_level
+ data[:, idx] = np.add(
+ obs_data[:, idx],
+ np.random.normal(0, 1, obs_data.shape[0]) * noise,
+ )
+
+ final_data[itrIdx] = data.T[~np.isnan(data.T)]
+
+ return final_data
+
+ # -------------------------------------------------------------------------
+ def _logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = spla.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(spla.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def _eval_model(self, samples=None, key='MAP'):
+ """
+ Evaluates Forward Model.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets. The default is None.
+ key : str, optional
+ Key string to be passed to the run_model_parallel method.
+ The default is 'MAP'.
+
+ Returns
+ -------
+ model_outputs : dict
+ Model outputs.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+
+ if samples is None:
+ self.samples = self.engine.ExpDesign.generate_samples(
+ self.n_samples, 'random')
+ else:
+ self.samples = samples
+ self.n_samples = len(samples)
+
+ model_outputs, _ = Model.run_model_parallel(
+ self.samples, key_str=key+self.name)
+
+ # Clean up
+ # Zip the subdirectories
+ try:
+ dir_name = f'{Model.name}MAP{self.name}'
+ key = dir_name + '_'
+ Model.zip_subdirs(dir_name, key)
+ except:
+ pass
+
+ return model_outputs
+
+ # -------------------------------------------------------------------------
+ def _kernel_rbf(self, X, hyperparameters):
+ """
+ Isotropic squared exponential kernel.
+
+ Higher l values lead to smoother functions and therefore to coarser
+ approximations of the training data. Lower l values make functions
+ more wiggly with wide uncertainty regions between training data points.
+
+ sigma_f controls the marginal variance of b(x)
+
+ Parameters
+ ----------
+ X : ndarray of shape (n_samples_X, n_features)
+
+ hyperparameters : Dict
+ Lambda characteristic length
+ sigma_f controls the marginal variance of b(x)
+ sigma_0 unresolvable error nugget term, interpreted as random
+ error that cannot be attributed to measurement error.
+ Returns
+ -------
+ var_cov_matrix : ndarray of shape (n_samples_X,n_samples_X)
+ Kernel k(X, X).
+
+ """
+ from sklearn.gaussian_process.kernels import RBF
+ min_max_scaler = preprocessing.MinMaxScaler()
+ X_minmax = min_max_scaler.fit_transform(X)
+
+ nparams = len(hyperparameters)
+ # characteristic length (0,1]
+ Lambda = hyperparameters[0]
+ # sigma_f controls the marginal variance of b(x)
+ sigma2_f = hyperparameters[1]
+
+ # cov_matrix = sigma2_f*rbf_kernel(X_minmax, gamma = 1/Lambda**2)
+
+ rbf = RBF(length_scale=Lambda)
+ cov_matrix = sigma2_f * rbf(X_minmax)
+ if nparams > 2:
+ # (unresolvable error) nugget term that is interpreted as random
+ # error that cannot be attributed to measurement error.
+ sigma2_0 = hyperparameters[2:]
+ for i, j in np.ndindex(cov_matrix.shape):
+ cov_matrix[i, j] += np.sum(sigma2_0) if i == j else 0
+
+ return cov_matrix
+
+ # -------------------------------------------------------------------------
+ def normpdf(self, outputs, obs_data, total_sigma2s, sigma2=None, std=None):
+ """
+ Calculates the likelihood of simulation outputs compared with
+ observation data.
+
+ Parameters
+ ----------
+ outputs : dict
+ A dictionary containing the simulation outputs as array of shape
+ (n_samples, n_measurement) for each model output.
+ obs_data : dict
+ A dictionary/dataframe containing the observation data.
+ total_sigma2s : dict
+ A dictionary with known values of the covariance diagonal entries,
+ a.k.a sigma^2.
+ sigma2 : array, optional
+ An array of the sigma^2 samples, when the covariance diagonal
+ entries are unknown and are being jointly inferred. The default is
+ None.
+ std : dict, optional
+ A dictionary containing the root mean squared error as array of
+ shape (n_samples, n_measurement) for each model output. The default
+ is None.
+
+ Returns
+ -------
+ logLik : array of shape (n_samples)
+ Likelihoods.
+
+ """
+ Model = self.engine.Model
+ logLik = 0.0
+
+ # Extract the requested model outputs for likelihood calulation
+ if self.req_outputs is None:
+ req_outputs = Model.Output.names
+ else:
+ req_outputs = list(self.req_outputs)
+
+ # Loop over the outputs
+ for idx, out in enumerate(req_outputs):
+
+ # (Meta)Model Output
+ nsamples, nout = outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Add the std of the PCE is chosen as emulator.
+ if self.emulator:
+ if std is not None:
+ tot_sigma2s += std[out]**2
+
+ # Covariance Matrix
+ covMatrix = np.diag(tot_sigma2s)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+
+ # If sigma2 is not given, use given total_sigma2s
+ if sigma2 is None:
+ logLik += stats.multivariate_normal.logpdf(
+ outputs[out][:, indices], data[indices], covMatrix)
+ continue
+
+ # Loop over each run/sample and calculate logLikelihood
+ logliks = np.zeros(nsamples)
+ for s_idx in range(nsamples):
+
+ # Simulation run
+ tot_outputs = outputs[out]
+
+ # Covariance Matrix
+ covMatrix = np.diag(tot_sigma2s)
+
+ if sigma2 is not None:
+ # Check the type error term
+ if hasattr(self, 'bias_inputs') and \
+ not hasattr(self, 'error_model'):
+ # Infer a Bias model usig Gaussian Process Regression
+ bias_inputs = np.hstack(
+ (self.bias_inputs[out],
+ tot_outputs[s_idx].reshape(-1, 1)))
+
+ params = sigma2[s_idx, idx*3:(idx+1)*3]
+ covMatrix = self._kernel_rbf(bias_inputs, params)
+ else:
+ # Infer equal sigma2s
+ try:
+ sigma_2 = sigma2[s_idx, idx]
+ except TypeError:
+ sigma_2 = 0.0
+
+ covMatrix += sigma_2 * np.eye(nout)
+ # covMatrix = np.diag(sigma2 * total_sigma2s)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+
+ # Compute loglikelihood
+ logliks[s_idx] = self._logpdf(
+ tot_outputs[s_idx, indices], data[indices], covMatrix
+ )
+
+ logLik += logliks
+ return logLik
+
+ # -------------------------------------------------------------------------
+ def _corr_factor_BME_old(self, Data, total_sigma2s, posterior):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ OrigModelOutput = self.engine.ExpDesign.Y
+ Model = self.engine.Model
+
+ # Posterior with guassian-likelihood
+ postDist = stats.gaussian_kde(posterior.T)
+
+ # Remove NaN
+ Data = Data[~np.isnan(Data)]
+ total_sigma2s = total_sigma2s[~np.isnan(total_sigma2s)]
+
+ # Covariance Matrix
+ covMatrix = np.diag(total_sigma2s[:self.n_tot_measurement])
+
+ # Extract the requested model outputs for likelihood calulation
+ if self.req_outputs is None:
+ OutputType = Model.Output.names
+ else:
+ OutputType = list(self.req_outputs)
+
+ # SampleSize = OrigModelOutput[OutputType[0]].shape[0]
+
+
+ # Flatten the OutputType for OrigModel
+ TotalOutputs = np.concatenate([OrigModelOutput[x] for x in OutputType], 1)
+
+ NrofBayesSamples = self.n_samples
+ # Evaluate MetaModel on the experimental design
+ Samples = self.engine.ExpDesign.X
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=Samples)
+
+ # Reset the NrofSamples to NrofBayesSamples
+ self.n_samples = NrofBayesSamples
+
+ # Flatten the OutputType for MetaModel
+ TotalPCEOutputs = np.concatenate([OutputRS[x] for x in OutputRS], 1)
+ TotalPCEstdOutputRS= np.concatenate([stdOutputRS[x] for x in stdOutputRS], 1)
+
+ logweight = 0
+ for i, sample in enumerate(Samples):
+ # Compute likelilhood output vs RS
+ covMatrix = np.diag(TotalPCEstdOutputRS[i]**2)
+ logLik = self._logpdf(TotalOutputs[i], TotalPCEOutputs[i], covMatrix)
+ # Compute posterior likelihood of the collocation points
+ logpostLik = np.log(postDist.pdf(sample[:, None]))[0]
+ if logpostLik != -np.inf:
+ logweight += logLik + logpostLik
+ return logweight
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = self.engine.ExpDesign.X
+ model_outputs = self.engine.ExpDesign.Y
+ Model = self.engine.Model
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ covMatrix = np.eye(len(y_m)) * 1/(2*np.pi)
+
+ # Select the data points to compare
+ try:
+ indices = self.selected_indices[out]
+ except:
+ indices = list(range(nout))
+ covMatrix = np.diag(covMatrix[indices, indices])
+ covMatrix_data = np.diag(covMatrix_data[indices, indices])
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self._logpdf(
+ y_m_hat[indices], data[indices],
+ covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self._logpdf(
+ y_m_hat[indices], y_m[indices],
+ covMatrix
+ )
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.mean(np.exp(logLik_model+logLik_data))
+
+ return np.log(weights)
+
+ # -------------------------------------------------------------------------
+ def _rejection_sampling(self):
+ """
+ Performs rejection sampling to update the prior distribution on the
+ input parameters.
+
+ Returns
+ -------
+ posterior : pandas.dataframe
+ Posterior samples of the input parameters.
+
+ """
+
+ MetaModel = self.MetaModel
+ try:
+ sigma2_prior = self.Discrepancy.sigma2_prior
+ except:
+ sigma2_prior = None
+
+ # Check if the discrepancy is defined as a distribution:
+ samples = self.samples
+
+ if sigma2_prior is not None:
+ samples = np.hstack((samples, sigma2_prior))
+
+ # Take the first column of Likelihoods (Observation data without noise)
+ if self.just_analysis or self.bayes_loocv:
+ index = self.n_tot_measurement-1
+ likelihoods = np.exp(self.log_likes[:, index], dtype=np.longdouble)#np.float128)
+ else:
+ likelihoods = np.exp(self.log_likes[:, 0], dtype=np.longdouble)#np.float128)
+
+ n_samples = len(likelihoods)
+ norm_ikelihoods = likelihoods / np.max(likelihoods)
+
+ # Normalize based on min if all Likelihoods are zero
+ if all(likelihoods == 0.0):
+ likelihoods = self.log_likes[:, 0]
+ norm_ikelihoods = likelihoods / np.min(likelihoods)
+
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, n_samples)[0]
+
+ # Reject the poorly performed prior
+ accepted_samples = samples[norm_ikelihoods >= unif]
+
+ # Output the Posterior
+ par_names = self.engine.ExpDesign.par_names
+ if sigma2_prior is not None:
+ for name in self.Discrepancy.name:
+ par_names.append(name)
+
+ return pd.DataFrame(accepted_samples, columns=sigma2_prior)
+
+ # -------------------------------------------------------------------------
+ def _posterior_predictive(self):
+ """
+ Stores the prior- and posterior predictive samples, i.e. model
+ evaluations using the samples, into hdf5 files.
+
+ priorPredictive.hdf5 : Prior predictive samples.
+ postPredictive_wo_noise.hdf5 : Posterior predictive samples without
+ the additive noise.
+ postPredictive.hdf5 : Posterior predictive samples with the additive
+ noise.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+
+ # Make a directory to save the prior/posterior predictive
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ os.makedirs(out_dir, exist_ok=True)
+
+ # Read observation data and perturb it if requested
+ if self.measured_data is None:
+ self.measured_data = Model.read_observation(case=self.name)
+
+ if not isinstance(self.measured_data, pd.DataFrame):
+ self.measured_data = pd.DataFrame(self.measured_data)
+
+ # X_values
+ x_values = self.engine.ExpDesign.x_values
+
+ try:
+ sigma2_prior = self.Discrepancy.sigma2_prior
+ except:
+ sigma2_prior = None
+
+ # Extract posterior samples
+ posterior_df = self.posterior_df
+
+ # Take care of the sigma2
+ if sigma2_prior is not None:
+ try:
+ sigma2s = posterior_df[self.Discrepancy.name].values
+ posterior_df = posterior_df.drop(
+ labels=self.Discrepancy.name, axis=1
+ )
+ except:
+ sigma2s = self.sigma2s
+
+ # Posterior predictive
+ if self.emulator:
+ if self.inference_method == 'rejection':
+ prior_pred = self.__mean_pce_prior_pred
+ if self.name.lower() != 'calib':
+ post_pred = self.__mean_pce_prior_pred
+ post_pred_std = self._std_pce_prior_pred
+ else:
+ post_pred, post_pred_std = MetaModel.eval_metamodel(
+ samples=posterior_df.values
+ )
+
+ else:
+ if self.inference_method == 'rejection':
+ prior_pred = self.__model_prior_pred
+ if self.name.lower() != 'calib':
+ post_pred = self.__mean_pce_prior_pred,
+ post_pred_std = self._std_pce_prior_pred
+ else:
+ post_pred = self._eval_model(
+ samples=posterior_df.values, key='PostPred'
+ )
+ # Correct the predictions with Model discrepancy
+ if hasattr(self, 'error_model') and self.error_model:
+ y_hat, y_std = self.error_MetaModel.eval_model_error(
+ self.bias_inputs, post_pred
+ )
+ post_pred, post_pred_std = y_hat, y_std
+
+ # Add discrepancy from likelihood Sample to the current posterior runs
+ total_sigma2 = self.Discrepancy.total_sigma2
+ post_pred_withnoise = copy.deepcopy(post_pred)
+ for varIdx, var in enumerate(Model.Output.names):
+ for i in range(len(post_pred[var])):
+ pred = post_pred[var][i]
+
+ # Known sigma2s
+ clean_sigma2 = total_sigma2[var][~np.isnan(total_sigma2[var])]
+ tot_sigma2 = clean_sigma2[:len(pred)]
+ cov = np.diag(tot_sigma2)
+
+ # Check the type error term
+ if sigma2_prior is not None:
+ # Inferred sigma2s
+ if hasattr(self, 'bias_inputs') and \
+ not hasattr(self, 'error_model'):
+ # TODO: Infer a Bias model usig GPR
+ bias_inputs = np.hstack((
+ self.bias_inputs[var], pred.reshape(-1, 1)))
+ params = sigma2s[i, varIdx*3:(varIdx+1)*3]
+ cov = self._kernel_rbf(bias_inputs, params)
+ else:
+ # Infer equal sigma2s
+ try:
+ sigma2 = sigma2s[i, varIdx]
+ except TypeError:
+ sigma2 = 0.0
+
+ # Convert biasSigma2s to a covMatrix
+ cov += sigma2 * np.eye(len(pred))
+
+ if self.emulator:
+ if hasattr(MetaModel, 'rmse') and \
+ MetaModel.rmse is not None:
+ stdPCE = MetaModel.rmse[var]
+ else:
+ stdPCE = post_pred_std[var][i]
+ # Expected value of variance (Assump: i.i.d stds)
+ cov += np.diag(stdPCE**2)
+
+ # Sample a multivariate normal distribution with mean of
+ # prediction and variance of cov
+ post_pred_withnoise[var][i] = np.random.multivariate_normal(
+ pred, cov, 1
+ )
+
+ # ----- Prior Predictive -----
+ if self.inference_method.lower() == 'rejection':
+ # Create hdf5 metadata
+ hdf5file = f'{out_dir}/priorPredictive.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=prior_pred[var])
+
+ # ----- Posterior Predictive only model evaluations -----
+ # Create hdf5 metadata
+ hdf5file = out_dir+'/postPredictive_wo_noise.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=post_pred[var])
+
+ # ----- Posterior Predictive with noise -----
+ # Create hdf5 metadata
+ hdf5file = out_dir+'/postPredictive.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ if hdf5_exist:
+ os.remove(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # Store x_values
+ if type(x_values) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grp_x_values.create_dataset(var, data=x_values[var])
+ else:
+ file.create_dataset("x_values", data=x_values)
+
+ # Store posterior predictive
+ grpY = file.create_group("EDY/")
+ for varIdx, var in enumerate(Model.Output.names):
+ grpY.create_dataset(var, data=post_pred_withnoise[var])
+
+ return
+
+ # -------------------------------------------------------------------------
+ def _plot_max_a_posteriori(self):
+ """
+ Plots the response of the model output against that of the metamodel at
+ the maximum a posteriori sample (mean or mode of posterior.)
+
+ Returns
+ -------
+ None.
+
+ """
+
+ MetaModel = self.MetaModel
+ Model = self.engine.Model
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ opt_sigma = self.Discrepancy.opt_sigma
+
+ # -------- Find MAP and run MetaModel and origModel --------
+ # Compute the MAP
+ if self.max_a_posteriori.lower() == 'mean':
+ if opt_sigma == "B":
+ Posterior_df = self.posterior_df.values
+ else:
+ Posterior_df = self.posterior_df.values[:, :-Model.n_outputs]
+ map_theta = Posterior_df.mean(axis=0).reshape(
+ (1, MetaModel.n_params))
+ else:
+ map_theta = stats.mode(Posterior_df.values, axis=0)[0]
+ # Prin report
+ print("\nPoint estimator:\n", map_theta[0])
+
+ # Run the models for MAP
+ # MetaModel
+ map_metamodel_mean, map_metamodel_std = MetaModel.eval_metamodel(
+ samples=map_theta)
+ self.map_metamodel_mean = map_metamodel_mean
+ self.map_metamodel_std = map_metamodel_std
+
+ # origModel
+ map_orig_model = self._eval_model(samples=map_theta)
+ self.map_orig_model = map_orig_model
+
+ # Extract slicing index
+ x_values = map_orig_model['x_values']
+
+ # List of markers and colors
+ Color = ['k', 'b', 'g', 'r']
+ Marker = 'x'
+
+ # Create a PdfPages object
+ pdf = PdfPages(f'./{out_dir}MAP_PCE_vs_Model_{self.name}.pdf')
+ fig = plt.figure()
+ for i, key in enumerate(Model.Output.names):
+
+ y_val = map_orig_model[key]
+ y_pce_val = map_metamodel_mean[key]
+ y_pce_val_std = map_metamodel_std[key]
+
+ plt.plot(x_values, y_val, color=Color[i], marker=Marker,
+ lw=2.0, label='$Y_{MAP}^{M}$')
+
+ plt.plot(
+ x_values, y_pce_val[i], color=Color[i], lw=2.0,
+ marker=Marker, linestyle='--', label='$Y_{MAP}^{PCE}$'
+ )
+ # plot the confidence interval
+ plt.fill_between(
+ x_values, y_pce_val[i] - 1.96*y_pce_val_std[i],
+ y_pce_val[i] + 1.96*y_pce_val_std[i],
+ color=Color[i], alpha=0.15
+ )
+
+ # Calculate the adjusted R_squared and RMSE
+ R2 = r2_score(y_pce_val.reshape(-1, 1), y_val.reshape(-1, 1))
+ rmse = np.sqrt(mean_squared_error(y_pce_val, y_val))
+
+ plt.ylabel(key)
+ plt.xlabel("Time [s]")
+ plt.title(f'Model vs MetaModel {key}')
+
+ ax = fig.axes[0]
+ leg = ax.legend(loc='best', frameon=True)
+ fig.canvas.draw()
+ p = leg.get_window_extent().inverse_transformed(ax.transAxes)
+ ax.text(
+ p.p0[1]-0.05, p.p1[1]-0.25,
+ f'RMSE = {rmse:.3f}\n$R^2$ = {R2:.3f}',
+ transform=ax.transAxes, color='black',
+ bbox=dict(facecolor='none', edgecolor='black',
+ boxstyle='round,pad=1'))
+
+ plt.show()
+
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ pdf.close()
+
+ # -------------------------------------------------------------------------
+ def _plot_post_predictive(self):
+ """
+ Plots the posterior predictives against the observation data.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ Model = self.engine.Model
+ out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ # Plot the posterior predictive
+ for out_idx, out_name in enumerate(Model.Output.names):
+ fig, ax = plt.subplots()
+ with sns.axes_style("ticks"):
+ x_key = list(self.measured_data)[0]
+
+ # --- Read prior and posterior predictive ---
+ if self.inference_method == 'rejection' and \
+ self.name.lower() != 'valid':
+ # --- Prior ---
+ # Load posterior predictive
+ f = h5py.File(
+ f'{out_dir}/priorPredictive.hdf5', 'r+')
+
+ try:
+ x_coords = np.array(f[f"x_values/{out_name}"])
+ except:
+ x_coords = np.array(f["x_values"])
+
+ X_values = np.repeat(x_coords, 10000)
+
+ prior_pred_df = {}
+ prior_pred_df[x_key] = X_values
+ prior_pred_df[out_name] = np.array(
+ f[f"EDY/{out_name}"])[:10000].flatten('F')
+ prior_pred_df = pd.DataFrame(prior_pred_df)
+
+ tags_post = ['prior'] * len(prior_pred_df)
+ prior_pred_df.insert(
+ len(prior_pred_df.columns), "Tags", tags_post,
+ True)
+ f.close()
+
+ # --- Posterior ---
+ f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+
+ X_values = np.repeat(
+ x_coords, np.array(f[f"EDY/{out_name}"]).shape[0])
+
+ post_pred_df = {}
+ post_pred_df[x_key] = X_values
+ post_pred_df[out_name] = np.array(
+ f[f"EDY/{out_name}"]).flatten('F')
+
+ post_pred_df = pd.DataFrame(post_pred_df)
+
+ tags_post = ['posterior'] * len(post_pred_df)
+ post_pred_df.insert(
+ len(post_pred_df.columns), "Tags", tags_post, True)
+ f.close()
+ # Concatenate two dataframes based on x_values
+ frames = [prior_pred_df, post_pred_df]
+ all_pred_df = pd.concat(frames)
+
+ # --- Plot posterior predictive ---
+ sns.violinplot(
+ x_key, y=out_name, data=all_pred_df, hue="Tags",
+ legend=False, ax=ax, split=True, inner=None,
+ color=".8")
+
+ # --- Plot Data ---
+ # Find the x,y coordinates for each point
+ x_coords = np.arange(x_coords.shape[0])
+ first_header = list(self.measured_data)[0]
+ obs_data = self.measured_data.round({first_header: 6})
+ sns.pointplot(
+ x=first_header, y=out_name, color='g', markers='x',
+ linestyles='', capsize=16, data=obs_data, ax=ax)
+
+ ax.errorbar(
+ x_coords, obs_data[out_name].values,
+ yerr=1.96*self.measurement_error[out_name],
+ ecolor='g', fmt=' ', zorder=-1)
+
+ # Add labels to the legend
+ handles, labels = ax.get_legend_handles_labels()
+ labels.append('Data')
+
+ data_marker = mlines.Line2D(
+ [], [], color='lime', marker='+', linestyle='None',
+ markersize=10)
+ handles.append(data_marker)
+
+ # Add legend
+ ax.legend(handles=handles, labels=labels, loc='best',
+ fontsize='large', frameon=True)
+
+ else:
+ # Load posterior predictive
+ f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+
+ try:
+ x_coords = np.array(f[f"x_values/{out_name}"])
+ except:
+ x_coords = np.array(f["x_values"])
+
+ mu = np.mean(np.array(f[f"EDY/{out_name}"]), axis=0)
+ std = np.std(np.array(f[f"EDY/{out_name}"]), axis=0)
+
+ # --- Plot posterior predictive ---
+ plt.plot(
+ x_coords, mu, marker='o', color='b',
+ label='Mean Post. Predictive')
+ plt.fill_between(
+ x_coords, mu-1.96*std, mu+1.96*std, color='b',
+ alpha=0.15)
+
+ # --- Plot Data ---
+ ax.plot(
+ x_coords, self.measured_data[out_name].values,
+ 'ko', label='data', markeredgecolor='w')
+
+ # --- Plot ExpDesign ---
+ orig_ED_Y = self.engine.ExpDesign.Y[out_name]
+ for output in orig_ED_Y:
+ plt.plot(
+ x_coords, output, color='grey', alpha=0.15
+ )
+
+ # Add labels for axes
+ plt.xlabel('Time [s]')
+ plt.ylabel(out_name)
+
+ # Add labels to the legend
+ handles, labels = ax.get_legend_handles_labels()
+
+ patch = Patch(color='b', alpha=0.15)
+ handles.insert(1, patch)
+ labels.insert(1, '95 $\\%$ CI')
+
+ # Add legend
+ ax.legend(handles=handles, labels=labels, loc='best',
+ frameon=True)
+
+ # Save figure in pdf format
+ if self.emulator:
+ plotname = f'/Post_Prior_Perd_{Model.name}_emulator'
+ else:
+ plotname = f'/Post_Prior_Perd_{Model.name}'
+
+ fig.savefig(f'./{out_dir}{plotname}_{out_name}.pdf',
+ bbox_inches='tight')
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_model_comparison.py b/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_model_comparison.py
new file mode 100644
index 0000000000000000000000000000000000000000..828613556e90ec0c529b91f2592eec148c98136b
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayes_inference/bayes_model_comparison.py
@@ -0,0 +1,654 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import os
+from scipy import stats
+import seaborn as sns
+import matplotlib.patches as patches
+import matplotlib.colors as mcolors
+import matplotlib.pylab as plt
+from .bayes_inference import BayesInference
+
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class BayesModelComparison:
+ """
+ A class to perform Bayesian Analysis.
+
+
+ Attributes
+ ----------
+ justifiability : bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `True`.
+ perturbed_data : array of shape (n_bootstrap_itrs, n_obs), optional
+ User defined perturbed data. The default is `None`.
+ n_bootstarp : int
+ Number of bootstrap iteration. The default is `1000`.
+ data_noise_level : float
+ A noise level to perturb the data set. The default is `0.01`.
+ just_n_meas : int
+ Number of measurements considered for visualization of the
+ justifiability results.
+
+ """
+
+ def __init__(self, justifiability=True, perturbed_data=None,
+ n_bootstarp=1000, data_noise_level=0.01, just_n_meas=2):
+
+ self.justifiability = justifiability
+ self.perturbed_data = perturbed_data
+ self.n_bootstarp = n_bootstarp
+ self.data_noise_level = data_noise_level
+ self.just_n_meas = just_n_meas
+
+ # --------------------------------------------------------------------------
+ def create_model_comparison(self, model_dict, opts_dict):
+ """
+ Starts the two-stage model comparison.
+ Stage I: Compare models using Bayes factors.
+ Stage II: Compare models via justifiability analysis.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ opts_dict : dict
+ A dictionary given the `BayesInference` options.
+
+ Example:
+
+ >>> opts_bootstrap = {
+ "bootstrap": True,
+ "n_samples": 10000,
+ "Discrepancy": DiscrepancyOpts,
+ "emulator": True,
+ "plot_post_pred": True
+ }
+
+ Returns
+ -------
+ output : dict
+ A dictionary containing the objects and the model weights for the
+ comparison using Bayes factors and justifiability analysis.
+
+ """
+
+ # Bayes factor
+ bayes_dict_bf, model_weights_dict_bf = self.compare_models(
+ model_dict, opts_dict
+ )
+
+ output = {
+ 'Bayes objects BF': bayes_dict_bf,
+ 'Model weights BF': model_weights_dict_bf
+ }
+
+ # Justifiability analysis
+ if self.justifiability:
+ bayes_dict_ja, model_weights_dict_ja = self.compare_models(
+ model_dict, opts_dict, justifiability=True
+ )
+
+ output['Bayes objects JA'] = bayes_dict_ja
+ output['Model weights JA'] = model_weights_dict_ja
+
+ return output
+
+ # --------------------------------------------------------------------------
+ def compare_models(self, model_dict, opts_dict, justifiability=False):
+ """
+ Passes the options to instantiates the BayesInference class for each
+ model and passes the options from `opts_dict`. Then, it starts the
+ computations.
+ It also creates a folder and saves the diagrams, e.g., Bayes factor
+ plot, confusion matrix, etc.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ opts_dict : dict
+ A dictionary given the `BayesInference` options.
+ justifiability : bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `False`.
+
+ Returns
+ -------
+ bayes_dict : dict
+ A dictionary with `BayesInference` objects.
+ model_weights_dict : dict
+ A dictionary containing the model weights.
+
+ """
+
+ if not isinstance(model_dict, dict):
+ raise Exception("To run model comparsion, you need to pass a "
+ "dictionary of models.")
+
+ # Extract model names
+ self.model_names = [*model_dict]
+
+ # Compute total number of the measurement points
+ Engine = list(model_dict.items())[0][1]
+ Engine.Model.read_observation()
+ self.n_meas = Engine.Model.n_obs
+
+ # ----- Generate data -----
+ # Find n_bootstrap
+ if self.perturbed_data is None:
+ n_bootstarp = self.n_bootstarp
+ else:
+ n_bootstarp = self.perturbed_data.shape[0]
+
+ # Create dataset
+ justData = self.generate_dataset(
+ model_dict, justifiability, n_bootstarp=n_bootstarp)
+
+ # Run create Interface for each model
+ bayes_dict = {}
+ for model in model_dict.keys():
+ print("-"*20)
+ print("Bayesian inference of {}.\n".format(model))
+
+ BayesOpts = BayesInference(model_dict[model])
+
+ # Set BayesInference options
+ for key, value in opts_dict.items():
+ if key in BayesOpts.__dict__.keys():
+ if key == "Discrepancy" and isinstance(value, dict):
+ setattr(BayesOpts, key, value[model])
+ else:
+ setattr(BayesOpts, key, value)
+
+ # Pass justifiability data as perturbed data
+ BayesOpts.perturbed_data = justData
+ BayesOpts.just_analysis = justifiability
+
+ bayes_dict[model] = BayesOpts.create_inference()
+ print("-"*20)
+
+ # Compute model weights
+ BME_Dict = dict()
+ for modelName, bayesObj in bayes_dict.items():
+ BME_Dict[modelName] = np.exp(bayesObj.log_BME, dtype=np.longdouble)#float128)
+
+ # BME correction in BayesInference class
+ model_weights = self.cal_model_weight(
+ BME_Dict, justifiability, n_bootstarp=n_bootstarp)
+
+ # Plot model weights
+ if justifiability:
+ model_names = self.model_names
+ model_names.insert(0, 'Observation')
+
+ # Split the model weights and save in a dict
+ list_ModelWeights = np.split(
+ model_weights, model_weights.shape[1]/self.n_meas, axis=1)
+ model_weights_dict = {key: weights for key, weights in
+ zip(model_names, list_ModelWeights)}
+
+ #self.plot_just_analysis(model_weights_dict)
+ else:
+ # Create box plot for model weights
+ self.plot_model_weights(model_weights, 'model_weights')
+
+ # Create kde plot for bayes factors
+ self.plot_bayes_factor(BME_Dict, 'kde_plot')
+
+ # Store model weights in a dict
+ model_weights_dict = {key: weights for key, weights in
+ zip(self.model_names, model_weights)}
+
+ return bayes_dict, model_weights_dict
+
+ # -------------------------------------------------------------------------
+ def generate_dataset(self, model_dict, justifiability=False,
+ n_bootstarp=1):
+ """
+ Generates the perturbed data set for the Bayes factor calculations and
+ the data set for the justifiability analysis.
+
+ Parameters
+ ----------
+ model_dict : dict
+ A dictionary including the metamodels.
+ bool, optional
+ Whether to perform the justifiability analysis. The default is
+ `False`.
+ n_bootstarp : int, optional
+ Number of bootstrap iterations. The default is `1`.
+
+ Returns
+ -------
+ all_just_data: array
+ Created data set.
+
+ """
+ # Compute some variables
+ all_just_data = []
+ Engine = list(model_dict.items())[0][1]
+ out_names = Engine.Model.Output.names
+
+ # Perturb observations for Bayes Factor
+ if self.perturbed_data is None:
+ self.perturbed_data = self.__perturb_data(
+ Engine.Model.observations, out_names, n_bootstarp,
+ noise_level=self.data_noise_level)
+
+ # Only for Bayes Factor
+ if not justifiability:
+ return self.perturbed_data
+
+ # Evaluate metamodel
+ runs = {}
+ for key, metaModel in model_dict.items():
+ y_hat, _ = metaModel.eval_metamodel(nsamples=n_bootstarp)
+ runs[key] = y_hat
+
+ # Generate data
+ for i in range(n_bootstarp):
+ y_data = self.perturbed_data[i].reshape(1, -1)
+ justData = np.tril(np.repeat(y_data, y_data.shape[1], axis=0))
+ # Use surrogate runs for data-generating process
+ for key, metaModel in model_dict.items():
+ model_data = np.array(
+ [runs[key][out][i] for out in out_names]).reshape(y_data.shape)
+ justData = np.vstack((
+ justData,
+ np.tril(np.repeat(model_data, model_data.shape[1], axis=0))
+ ))
+ # Save in a list
+ all_just_data.append(justData)
+
+ # Squeeze the array
+ all_just_data = np.array(all_just_data).transpose(1, 0, 2).reshape(
+ -1, np.array(all_just_data).shape[2]
+ )
+
+ return all_just_data
+
+ # -------------------------------------------------------------------------
+ def __perturb_data(self, data, output_names, n_bootstrap, noise_level):
+ """
+ Returns an array with n_bootstrap_itrs rowsof perturbed data.
+ The first row includes the original observation data.
+ If `self.bayes_loocv` is True, a 2d-array will be returned with
+ repeated rows and zero diagonal entries.
+
+ Parameters
+ ----------
+ data : pandas DataFrame
+ Observation data.
+ output_names : list
+ List of the output names.
+
+ Returns
+ -------
+ final_data : array
+ Perturbed data set.
+
+ """
+ obs_data = data[output_names].values
+ n_measurement, n_outs = obs_data.shape
+ n_tot_measurement = obs_data[~np.isnan(obs_data)].shape[0]
+ final_data = np.zeros(
+ (n_bootstrap, n_tot_measurement)
+ )
+ final_data[0] = obs_data.T[~np.isnan(obs_data.T)]
+ for itrIdx in range(1, n_bootstrap):
+ data = np.zeros((n_measurement, n_outs))
+ for idx in range(len(output_names)):
+ std = np.nanstd(obs_data[:, idx])
+ if std == 0:
+ std = 0.001
+ noise = std * noise_level
+ data[:, idx] = np.add(
+ obs_data[:, idx],
+ np.random.normal(0, 1, obs_data.shape[0]) * noise,
+ )
+
+ final_data[itrIdx] = data.T[~np.isnan(data.T)]
+
+ return final_data
+
+ # -------------------------------------------------------------------------
+ def cal_model_weight(self, BME_Dict, justifiability=False, n_bootstarp=1):
+ """
+ Normalize the BME (Asumption: Model Prior weights are equal for models)
+
+ Parameters
+ ----------
+ BME_Dict : dict
+ A dictionary containing the BME values.
+
+ Returns
+ -------
+ model_weights : array
+ Model weights.
+
+ """
+ # Stack the BME values for all models
+ all_BME = np.vstack(list(BME_Dict.values()))
+
+ if justifiability:
+ # Compute expected log_BME for justifiabiliy analysis
+ all_BME = all_BME.reshape(
+ all_BME.shape[0], -1, n_bootstarp).mean(axis=2)
+
+ # Model weights
+ model_weights = np.divide(all_BME, np.nansum(all_BME, axis=0))
+
+ return model_weights
+
+ # -------------------------------------------------------------------------
+ def plot_just_analysis(self, model_weights_dict):
+ """
+ Visualizes the confusion matrix and the model wights for the
+ justifiability analysis.
+
+ Parameters
+ ----------
+ model_weights_dict : dict
+ Model weights.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+ Color = [*mcolors.TABLEAU_COLORS]
+ names = [*model_weights_dict]
+
+ model_names = [model.replace('_', '$-$') for model in self.model_names]
+ for name in names:
+ fig, ax = plt.subplots()
+ for i, model in enumerate(model_names[1:]):
+ plt.plot(list(range(1, self.n_meas+1)),
+ model_weights_dict[name][i],
+ color=Color[i], marker='o',
+ ms=10, linewidth=2, label=model
+ )
+
+ plt.title(f"Data generated by: {name.replace('_', '$-$')}")
+ plt.ylabel("Weights")
+ plt.xlabel("No. of measurement points")
+ ax.set_xticks(list(range(1, self.n_meas+1)))
+ plt.legend(loc="best")
+ fig.savefig(
+ f'{directory}modelWeights_{name}.svg', bbox_inches='tight'
+ )
+ plt.close()
+
+ # Confusion matrix for some measurement points
+ epsilon = 1 if self.just_n_meas != 1 else 0
+ for index in range(0, self.n_meas+epsilon, self.just_n_meas):
+ weights = np.array(
+ [model_weights_dict[key][:, index] for key in model_weights_dict]
+ )
+ g = sns.heatmap(
+ weights.T, annot=True, cmap='Blues', xticklabels=model_names,
+ yticklabels=model_names[1:], annot_kws={"size": 24}
+ )
+
+ # x axis on top
+ g.xaxis.tick_top()
+ g.xaxis.set_label_position('top')
+ g.set_xlabel(r"\textbf{Data generated by:}", labelpad=15)
+ g.set_ylabel(r"\textbf{Model weight for:}", labelpad=15)
+ g.figure.savefig(
+ f"{directory}confusionMatrix_ND_{index+1}.pdf",
+ bbox_inches='tight'
+ )
+ plt.close()
+
+ # -------------------------------------------------------------------------
+ def plot_model_weights(self, model_weights, plot_name):
+ """
+ Visualizes the model weights resulting from BMS via the observation
+ data.
+
+ Parameters
+ ----------
+ model_weights : array
+ Model weights.
+ plot_name : str
+ Plot name.
+
+ Returns
+ -------
+ None.
+
+ """
+ font_size = 40
+ # mkdir for plots
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+
+ # Create figure
+ fig, ax = plt.subplots()
+
+ # Filter data using np.isnan
+ mask = ~np.isnan(model_weights.T)
+ filtered_data = [d[m] for d, m in zip(model_weights, mask.T)]
+
+ # Create the boxplot
+ bp = ax.boxplot(filtered_data, patch_artist=True, showfliers=False)
+
+ # change outline color, fill color and linewidth of the boxes
+ for box in bp['boxes']:
+ # change outline color
+ box.set(color='#7570b3', linewidth=4)
+ # change fill color
+ box.set(facecolor='#1b9e77')
+
+ # change color and linewidth of the whiskers
+ for whisker in bp['whiskers']:
+ whisker.set(color='#7570b3', linewidth=2)
+
+ # change color and linewidth of the caps
+ for cap in bp['caps']:
+ cap.set(color='#7570b3', linewidth=2)
+
+ # change color and linewidth of the medians
+ for median in bp['medians']:
+ median.set(color='#b2df8a', linewidth=2)
+
+ # change the style of fliers and their fill
+ # for flier in bp['fliers']:
+ # flier.set(marker='o', color='#e7298a', alpha=0.75)
+
+ # Custom x-axis labels
+ model_names = [model.replace('_', '$-$') for model in self.model_names]
+ ax.set_xticklabels(model_names)
+
+ ax.set_ylabel('Weight', fontsize=font_size)
+
+ # Title
+ plt.title('Posterior Model Weights')
+
+ # Set y lim
+ ax.set_ylim((-0.05, 1.05))
+
+ # Set size of the ticks
+ for t in ax.get_xticklabels():
+ t.set_fontsize(font_size)
+ for t in ax.get_yticklabels():
+ t.set_fontsize(font_size)
+
+ # Save the figure
+ fig.savefig(
+ f'./{directory}{plot_name}.pdf', bbox_inches='tight'
+ )
+
+ plt.close()
+
+ # -------------------------------------------------------------------------
+ def plot_bayes_factor(self, BME_Dict, plot_name=''):
+ """
+ Plots the Bayes factor distibutions in a :math:`N_m \\times N_m`
+ matrix, where :math:`N_m` is the number of the models.
+
+ Parameters
+ ----------
+ BME_Dict : dict
+ A dictionary containing the BME values of the models.
+ plot_name : str, optional
+ Plot name. The default is ''.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ font_size = 40
+
+ # mkdir for plots
+ directory = 'Outputs_Comparison/'
+ os.makedirs(directory, exist_ok=True)
+
+ Colors = ["blue", "green", "gray", "brown"]
+
+ model_names = list(BME_Dict.keys())
+ nModels = len(model_names)
+
+ # Plots
+ fig, axes = plt.subplots(
+ nrows=nModels, ncols=nModels, sharex=True, sharey=True
+ )
+
+ for i, key_i in enumerate(model_names):
+
+ for j, key_j in enumerate(model_names):
+ ax = axes[i, j]
+ # Set size of the ticks
+ for t in ax.get_xticklabels():
+ t.set_fontsize(font_size)
+ for t in ax.get_yticklabels():
+ t.set_fontsize(font_size)
+
+ if j != i:
+
+ # Null hypothesis: key_j is the better model
+ BayesFactor = np.log10(
+ np.divide(BME_Dict[key_i], BME_Dict[key_j])
+ )
+
+ # sns.kdeplot(BayesFactor, ax=ax, color=Colors[i], shade=True)
+ # sns.histplot(BayesFactor, ax=ax, stat="probability",
+ # kde=True, element='step',
+ # color=Colors[j])
+
+ # taken from seaborn's source code (utils.py and
+ # distributions.py)
+ def seaborn_kde_support(data, bw, gridsize, cut, clip):
+ if clip is None:
+ clip = (-np.inf, np.inf)
+ support_min = max(data.min() - bw * cut, clip[0])
+ support_max = min(data.max() + bw * cut, clip[1])
+ return np.linspace(support_min, support_max, gridsize)
+
+ kde_estim = stats.gaussian_kde(
+ BayesFactor, bw_method='scott'
+ )
+
+ # manual linearization of data
+ # linearized = np.linspace(
+ # quotient.min(), quotient.max(), num=500)
+
+ # or better: mimic seaborn's internal stuff
+ bw = kde_estim.scotts_factor() * np.std(BayesFactor)
+ linearized = seaborn_kde_support(
+ BayesFactor, bw, 100, 3, None)
+
+ # computes values of the estimated function on the
+ # estimated linearized inputs
+ Z = kde_estim.evaluate(linearized)
+
+ # https://stackoverflow.com/questions/29661574/normalize-
+ # numpy-array-columns-in-python
+ def normalize(x):
+ return (x - x.min(0)) / x.ptp(0)
+
+ # normalize so it is between 0;1
+ Z2 = normalize(Z)
+ ax.plot(linearized, Z2, "-", color=Colors[i], linewidth=4)
+ ax.fill_between(
+ linearized, 0, Z2, color=Colors[i], alpha=0.25
+ )
+
+ # Draw BF significant levels according to Jeffreys 1961
+ # Strong evidence for both models
+ ax.axvline(
+ x=np.log10(3), ymin=0, linewidth=4, color='dimgrey'
+ )
+ # Strong evidence for one model
+ ax.axvline(
+ x=np.log10(10), ymin=0, linewidth=4, color='orange'
+ )
+ # Decisive evidence for one model
+ ax.axvline(
+ x=np.log10(100), ymin=0, linewidth=4, color='r'
+ )
+
+ # legend
+ BF_label = key_i.replace('_', '$-$') + \
+ '/' + key_j.replace('_', '$-$')
+ legend_elements = [
+ patches.Patch(facecolor=Colors[i], edgecolor=Colors[i],
+ label=f'BF({BF_label})')
+ ]
+ ax.legend(
+ loc='upper left', handles=legend_elements,
+ fontsize=font_size-(nModels+1)*5
+ )
+
+ elif j == i:
+ # build a rectangle in axes coords
+ left, width = 0, 1
+ bottom, height = 0, 1
+
+ # axes coordinates are 0,0 is bottom left and 1,1 is upper
+ # right
+ p = patches.Rectangle(
+ (left, bottom), width, height, color='white',
+ fill=True, transform=ax.transAxes, clip_on=False
+ )
+ ax.grid(False)
+ ax.add_patch(p)
+ # ax.text(0.5*(left+right), 0.5*(bottom+top), key_i,
+ fsize = font_size+20 if nModels < 4 else font_size
+ ax.text(0.5, 0.5, key_i.replace('_', '$-$'),
+ horizontalalignment='center',
+ verticalalignment='center',
+ fontsize=fsize, color=Colors[i],
+ transform=ax.transAxes)
+
+ # Defining custom 'ylim' values.
+ custom_ylim = (0, 1.05)
+
+ # Setting the values for all axes.
+ plt.setp(axes, ylim=custom_ylim)
+
+ # set labels
+ for i in range(nModels):
+ axes[-1, i].set_xlabel('log$_{10}$(BF)', fontsize=font_size)
+ axes[i, 0].set_ylabel('Probability', fontsize=font_size)
+
+ # Adjust subplots
+ plt.subplots_adjust(wspace=0.2, hspace=0.1)
+
+ plt.savefig(
+ f'./{directory}Bayes_Factor{plot_name}.pdf', bbox_inches='tight'
+ )
+
+ plt.close()
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/discrepancy.py b/examples/model-comparison/bayesvalidrox/bayes_inference/discrepancy.py
new file mode 100644
index 0000000000000000000000000000000000000000..fff32a2500ae20b3667c7b0ec2cc85c1da614688
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayes_inference/discrepancy.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import scipy.stats as stats
+from bayesvalidrox.surrogate_models.exp_designs import ExpDesigns
+
+
+class Discrepancy:
+ """
+ Discrepancy class for Bayesian inference method.
+ We define the reference or reality to be equal to what we can model and a
+ descripancy term \\( \\epsilon \\). We consider the followin format:
+
+ $$\\textbf{y}_{\\text{reality}} = \\mathcal{M}(\\theta) + \\epsilon,$$
+
+ where \\( \\epsilon \\in R^{N_{out}} \\) represents the the effects of
+ measurement error and model inaccuracy. For simplicity, it can be defined
+ as an additive Gaussian disrepancy with zeromean and given covariance
+ matrix \\( \\Sigma \\):
+
+ $$\\epsilon \\sim \\mathcal{N}(\\epsilon|0, \\Sigma). $$
+
+ In the context of model inversion or calibration, an observation point
+ \\( \\textbf{y}_i \\in \\mathcal{y} \\) is a realization of a Gaussian
+ distribution with mean value of \\(\\mathcal{M}(\\theta) \\) and covariance
+ matrix of \\( \\Sigma \\).
+
+ $$ p(\\textbf{y}|\\theta) = \\mathcal{N}(\\textbf{y}|\\mathcal{M}
+ (\\theta))$$
+
+ The following options are available:
+
+ * Option A: With known redidual covariance matrix \\(\\Sigma\\) for
+ independent measurements.
+
+ * Option B: With unknown redidual covariance matrix \\(\\Sigma\\),
+ paramethrized as \\(\\Sigma(\\theta_{\\epsilon})=\\sigma^2 \\textbf{I}_
+ {N_{out}}\\) with unknown residual variances \\(\\sigma^2\\).
+ This term will be jointly infer with the uncertain input parameters. For
+ the inversion, you need to define a prior marginal via `Input` class. Note
+ that \\(\\sigma^2\\) is only a single scalar multiplier for the diagonal
+ entries of the covariance matrix \\(\\Sigma\\).
+
+ Attributes
+ ----------
+ InputDisc : obj
+ Input object. When the \\(\\sigma^2\\) is expected to be inferred
+ jointly with the parameters (`Option B`).If multiple output groups are
+ defined by `Model.Output.names`, each model output needs to have.
+ a prior marginal using the `Input` class. The default is `''`.
+ disc_type : str
+ Type of the noise definition. `'Gaussian'` is only supported so far.
+ parameters : dict or pandas.DataFrame
+ Known residual variance \\(\\sigma^2\\), i.e. diagonal entry of the
+ covariance matrix of the multivariate normal likelihood in case of
+ `Option A`.
+
+ """
+
+ def __init__(self, InputDisc='', disc_type='Gaussian', parameters=None):
+ self.InputDisc = InputDisc
+ self.disc_type = disc_type
+ self.parameters = parameters
+
+ # -------------------------------------------------------------------------
+ def get_sample(self, n_samples):
+ """
+ Generate samples for the \\(\\sigma^2\\), i.e. the diagonal entries of
+ the variance-covariance matrix in the multivariate normal distribution.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of samples (parameter sets).
+
+ Returns
+ -------
+ sigma2_prior: array of shape (n_samples, n_params)
+ \\(\\sigma^2\\) samples.
+
+ """
+ self.n_samples = n_samples # TODO: not used again in here - needed from the outside?
+
+ if self.InputDisc == '':
+ raise AttributeError('Cannot create new samples, please provide input distributions')
+
+ # Create and store BoundTuples
+ self.ExpDesign = ExpDesigns(self.InputDisc)
+ self.ExpDesign.sampling_method = 'random'
+ self.ExpDesign.generate_ED(
+ n_samples, max_pce_deg=1
+ )
+ # TODO: need to recheck the following line
+ # This used to simply be the return from the call above
+ self.sigma2_prior = self.ExpDesign.X
+
+ # Naive approach: Fit a gaussian kernel to the provided data
+ self.ExpDesign.JDist = stats.gaussian_kde(self.ExpDesign.raw_data)
+
+ # Save the names of sigmas
+ if len(self.InputDisc.Marginals) != 0:
+ self.name = []
+ for Marginalidx in range(len(self.InputDisc.Marginals)):
+ self.name.append(self.InputDisc.Marginals[Marginalidx].name)
+
+ return self.sigma2_prior
diff --git a/examples/model-comparison/bayesvalidrox/bayes_inference/mcmc.py b/examples/model-comparison/bayesvalidrox/bayes_inference/mcmc.py
new file mode 100644
index 0000000000000000000000000000000000000000..fe22a152f117aab7023bfe6592ce3a48bb0b3aec
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayes_inference/mcmc.py
@@ -0,0 +1,909 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import numpy as np
+import emcee
+import pandas as pd
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_pdf import PdfPages
+import multiprocessing
+import scipy.stats as st
+from scipy.linalg import cholesky as chol
+import warnings
+import shutil
+os.environ["OMP_NUM_THREADS"] = "1"
+
+
+class MCMC:
+ """
+ A class for bayesian inference via a Markov-Chain Monte-Carlo (MCMC)
+ Sampler to approximate the posterior distribution of the Bayes theorem:
+ $$p(\\theta|\\mathcal{y}) = \\frac{p(\\mathcal{y}|\\theta) p(\\theta)}
+ {p(\\mathcal{y})}.$$
+
+ This class make inference with emcee package [1] using an Affine Invariant
+ Ensemble sampler (AIES) [2].
+
+ [1] Foreman-Mackey, D., Hogg, D.W., Lang, D. and Goodman, J., 2013.emcee:
+ the MCMC hammer. Publications of the Astronomical Society of the
+ Pacific, 125(925), p.306. https://emcee.readthedocs.io/en/stable/
+
+ [2] Goodman, J. and Weare, J., 2010. Ensemble samplers with affine
+ invariance. Communications in applied mathematics and computational
+ science, 5(1), pp.65-80.
+
+
+ Attributes
+ ----------
+ BayesOpts : obj
+ Bayes object.
+ """
+
+ def __init__(self, BayesOpts):
+
+ self.BayesOpts = BayesOpts
+
+ def run_sampler(self, observation, total_sigma2):
+
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.engine.MetaModel
+ Model = BayesObj.engine.Model
+ Discrepancy = self.BayesOpts.Discrepancy
+ n_cpus = Model.n_cpus
+ priorDist = BayesObj.engine.ExpDesign.JDist
+ ndim = MetaModel.n_params
+ self.counter = 0
+ output_dir = f'Outputs_Bayes_{Model.name}_{self.BayesOpts.name}'
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
+ self.observation = observation
+ self.total_sigma2 = total_sigma2
+
+ # Unpack mcmc parameters given to BayesObj.mcmc_params
+ self.initsamples = None
+ self.nwalkers = 100
+ self.nburn = 200
+ self.nsteps = 100000
+ self.moves = None
+ self.mp = False
+ self.verbose = False
+
+ # Extract initial samples
+ if 'init_samples' in BayesObj.mcmc_params:
+ self.initsamples = BayesObj.mcmc_params['init_samples']
+ if isinstance(self.initsamples, pd.DataFrame):
+ self.initsamples = self.initsamples.values
+
+ # Extract number of steps per walker
+ if 'n_steps' in BayesObj.mcmc_params:
+ self.nsteps = int(BayesObj.mcmc_params['n_steps'])
+ # Extract number of walkers (chains)
+ if 'n_walkers' in BayesObj.mcmc_params:
+ self.nwalkers = int(BayesObj.mcmc_params['n_walkers'])
+ # Extract moves
+ if 'moves' in BayesObj.mcmc_params:
+ self.moves = BayesObj.mcmc_params['moves']
+ # Extract multiprocessing
+ if 'multiprocessing' in BayesObj.mcmc_params:
+ self.mp = BayesObj.mcmc_params['multiprocessing']
+ # Extract verbose
+ if 'verbose' in BayesObj.mcmc_params:
+ self.verbose = BayesObj.mcmc_params['verbose']
+
+ # Set initial samples
+ np.random.seed(0)
+ if self.initsamples is None:
+ try:
+ initsamples = priorDist.sample(self.nwalkers).T
+ except:
+ # when aPCE selected - gaussian kernel distribution
+ inputSamples = MetaModel.ExpDesign.raw_data.T
+ random_indices = np.random.choice(
+ len(inputSamples), size=self.nwalkers, replace=False
+ )
+ initsamples = inputSamples[random_indices]
+
+ else:
+ if self.initsamples.ndim == 1:
+ # When MAL is given.
+ theta = self.initsamples
+ initsamples = [theta + 1e-1*np.multiply(
+ np.random.randn(ndim), theta) for i in
+ range(self.nwalkers)]
+ else:
+ # Pick samples based on a uniform dist between min and max of
+ # each dim
+ initsamples = np.zeros((self.nwalkers, ndim))
+ bound_tuples = []
+ for idx_dim in range(ndim):
+ lower = np.min(self.initsamples[:, idx_dim])
+ upper = np.max(self.initsamples[:, idx_dim])
+ bound_tuples.append((lower, upper))
+ dist = st.uniform(loc=lower, scale=upper-lower)
+ initsamples[:, idx_dim] = dist.rvs(size=self.nwalkers)
+
+ # Update lower and upper
+ MetaModel.ExpDesign.bound_tuples = bound_tuples
+
+ # Check if sigma^2 needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ sigma2_samples = Discrepancy.get_sample(self.nwalkers)
+
+ # Update initsamples
+ initsamples = np.hstack((initsamples, sigma2_samples))
+
+ # Update ndim
+ ndim = initsamples.shape[1]
+
+ # Discrepancy bound
+ disc_bound_tuple = Discrepancy.ExpDesign.bound_tuples
+
+ # Update bound_tuples
+ BayesObj.engine.ExpDesign.bound_tuples += disc_bound_tuple
+
+ print("\n>>>> Bayesian inference with MCMC for "
+ f"{self.BayesOpts.name} started. <<<<<<")
+
+ # Set up the backend
+ filename = f"{output_dir}/emcee_sampler.h5"
+ backend = emcee.backends.HDFBackend(filename)
+ # Clear the backend in case the file already exists
+ backend.reset(self.nwalkers, ndim)
+
+ # Define emcee sampler
+ # Here we'll set up the computation. emcee combines multiple "walkers",
+ # each of which is its own MCMC chain. The number of trace results will
+ # be nwalkers * nsteps.
+ if self.mp:
+ # Run in parallel
+ if n_cpus is None:
+ n_cpus = multiprocessing.cpu_count()
+
+ with multiprocessing.Pool(n_cpus) as pool:
+ sampler = emcee.EnsembleSampler(
+ self.nwalkers, ndim, self.log_posterior, moves=self.moves,
+ pool=pool, backend=backend
+ )
+
+ # Check if a burn-in phase is needed!
+ if self.initsamples is None:
+ # Burn-in
+ print("\n Burn-in period is starting:")
+ pos = sampler.run_mcmc(
+ initsamples, self.nburn, progress=True
+ )
+
+ # Reset sampler
+ sampler.reset()
+ pos = pos.coords
+ else:
+ pos = initsamples
+
+ # Production run
+ print("\n Production run is starting:")
+ pos, prob, state = sampler.run_mcmc(
+ pos, self.nsteps, progress=True
+ )
+
+ else:
+ # Run in series and monitor the convergence
+ sampler = emcee.EnsembleSampler(
+ self.nwalkers, ndim, self.log_posterior, moves=self.moves,
+ backend=backend, vectorize=True
+ )
+
+ # Check if a burn-in phase is needed!
+ if self.initsamples is None:
+ # Burn-in
+ print("\n Burn-in period is starting:")
+ pos = sampler.run_mcmc(
+ initsamples, self.nburn, progress=True
+ )
+
+ # Reset sampler
+ sampler.reset()
+ pos = pos.coords
+ else:
+ pos = initsamples
+
+ # Production run
+ print("\n Production run is starting:")
+
+ # Track how the average autocorrelation time estimate changes
+ autocorrIdx = 0
+ autocorr = np.empty(self.nsteps)
+ tauold = np.inf
+ autocorreverynsteps = 50
+
+ # sample step by step using the generator sampler.sample
+ for sample in sampler.sample(pos,
+ iterations=self.nsteps,
+ tune=True,
+ progress=True):
+
+ # only check convergence every autocorreverynsteps steps
+ if sampler.iteration % autocorreverynsteps:
+ continue
+
+ # Train model discrepancy/error
+ if hasattr(BayesObj, 'errorModel') and BayesObj.errorModel \
+ and not sampler.iteration % 3 * autocorreverynsteps:
+ try:
+ self.error_MetaModel = self.train_error_model(sampler)
+ except:
+ pass
+
+ # Print the current mean acceptance fraction
+ if self.verbose:
+ print("\nStep: {}".format(sampler.iteration))
+ acc_fr = np.mean(sampler.acceptance_fraction)
+ print(f"Mean acceptance fraction: {acc_fr:.3f}")
+
+ # compute the autocorrelation time so far
+ # using tol=0 means that we'll always get an estimate even if
+ # it isn't trustworthy
+ tau = sampler.get_autocorr_time(tol=0)
+ # average over walkers
+ autocorr[autocorrIdx] = np.nanmean(tau)
+ autocorrIdx += 1
+
+ # output current autocorrelation estimate
+ if self.verbose:
+ print(f"Mean autocorr. time estimate: {np.nanmean(tau):.3f}")
+ list_gr = np.round(self.gelman_rubin(sampler.chain), 3)
+ print("Gelman-Rubin Test*: ", list_gr)
+
+ # check convergence
+ converged = np.all(tau*autocorreverynsteps < sampler.iteration)
+ converged &= np.all(np.abs(tauold - tau) / tau < 0.01)
+ converged &= np.all(self.gelman_rubin(sampler.chain) < 1.1)
+
+ if converged:
+ break
+ tauold = tau
+
+ # Posterior diagnostics
+ try:
+ tau = sampler.get_autocorr_time(tol=0)
+ except emcee.autocorr.AutocorrError:
+ tau = 5
+
+ if all(np.isnan(tau)):
+ tau = 5
+
+ burnin = int(2*np.nanmax(tau))
+ thin = int(0.5*np.nanmin(tau)) if int(0.5*np.nanmin(tau)) != 0 else 1
+ finalsamples = sampler.get_chain(discard=burnin, flat=True, thin=thin)
+ acc_fr = np.nanmean(sampler.acceptance_fraction)
+ list_gr = np.round(self.gelman_rubin(sampler.chain[:, burnin:]), 3)
+
+ # Print summary
+ print('\n')
+ print('-'*15 + 'Posterior diagnostics' + '-'*15)
+ print(f"Mean auto-correlation time: {np.nanmean(tau):.3f}")
+ print(f"Thin: {thin}")
+ print(f"Burn-in: {burnin}")
+ print(f"Flat chain shape: {finalsamples.shape}")
+ print(f"Mean acceptance fraction*: {acc_fr:.3f}")
+ print("Gelman-Rubin Test**: ", list_gr)
+
+ print("\n* This value must lay between 0.234 and 0.5.")
+ print("** These values must be smaller than 1.1.")
+ print('-'*50)
+
+ print(f"\n>>>> Bayesian inference with MCMC for {self.BayesOpts.name} "
+ "successfully completed. <<<<<<\n")
+
+ # Extract parameter names and their prior ranges
+ par_names = self.BayesOpts.engine.ExpDesign.par_names
+
+ if Discrepancy.opt_sigma != 'B':
+ for i in range(len(Discrepancy.InputDisc.Marginals)):
+ par_names.append(Discrepancy.InputDisc.Marginals[i].name)
+
+ params_range = self.BayesOpts.engine.ExpDesign.bound_tuples
+
+ # Plot traces
+ if self.verbose and self.nsteps < 10000:
+ pdf = PdfPages(output_dir+'/traceplots.pdf')
+ fig = plt.figure()
+ for parIdx in range(ndim):
+ # Set up the axes with gridspec
+ fig = plt.figure()
+ grid = plt.GridSpec(4, 4, hspace=0.2, wspace=0.2)
+ main_ax = fig.add_subplot(grid[:-1, :3])
+ y_hist = fig.add_subplot(grid[:-1, -1], xticklabels=[],
+ sharey=main_ax)
+
+ for i in range(self.nwalkers):
+ samples = sampler.chain[i, :, parIdx]
+ main_ax.plot(samples, '-')
+
+ # histogram on the attached axes
+ y_hist.hist(samples[burnin:], 40, histtype='stepfilled',
+ orientation='horizontal', color='gray')
+
+ main_ax.set_ylim(params_range[parIdx])
+ main_ax.set_title('traceplot for ' + par_names[parIdx])
+ main_ax.set_xlabel('step number')
+
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ pdf.close()
+
+ # plot development of autocorrelation estimate
+ if not self.mp:
+ fig1 = plt.figure()
+ steps = autocorreverynsteps*np.arange(1, autocorrIdx+1)
+ taus = autocorr[:autocorrIdx]
+ plt.plot(steps, steps / autocorreverynsteps, "--k")
+ plt.plot(steps, taus)
+ plt.xlim(0, steps.max())
+ plt.ylim(0, np.nanmax(taus)+0.1*(np.nanmax(taus)-np.nanmin(taus)))
+ plt.xlabel("number of steps")
+ plt.ylabel(r"mean $\hat{\tau}$")
+ fig1.savefig(f"{output_dir}/autocorrelation_time.pdf",
+ bbox_inches='tight')
+
+ # logml_dict = self.marginal_llk_emcee(sampler, self.nburn, logp=None,
+ # maxiter=5000)
+ # print('\nThe Bridge Sampling Estimation is "
+ # f"{logml_dict['logml']:.5f}.')
+
+ # # Posterior-based expectation of posterior probablity
+ # postExpPostLikelihoods = np.mean(sampler.get_log_prob(flat=True)
+ # [self.nburn*self.nwalkers:])
+
+ # # Posterior-based expectation of prior densities
+ # postExpPrior = np.mean(self.log_prior(emcee_trace.T))
+
+ # # Posterior-based expectation of likelihoods
+ # postExpLikelihoods_emcee = postExpPostLikelihoods - postExpPrior
+
+ # # Calculate Kullback-Leibler Divergence
+ # KLD_emcee = postExpLikelihoods_emcee - logml_dict['logml']
+ # print("Kullback-Leibler divergence: %.5f"%KLD_emcee)
+
+ # # Information Entropy based on Entropy paper Eq. 38
+ # infEntropy_emcee = logml_dict['logml'] - postExpPrior -
+ # postExpLikelihoods_emcee
+ # print("Information Entropy: %.5f" %infEntropy_emcee)
+
+ Posterior_df = pd.DataFrame(finalsamples, columns=par_names)
+
+ return Posterior_df
+
+ # -------------------------------------------------------------------------
+ def log_prior(self, theta):
+ """
+ Calculates the log prior likelihood \\( p(\\theta)\\) for the given
+ parameter set(s) \\( \\theta \\).
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter sets, i.e. proposals of MCMC chains.
+
+ Returns
+ -------
+ logprior: float or array of shape n_samples
+ Log prior likelihood. If theta has only one row, a single value is
+ returned otherwise an array.
+
+ """
+
+ MetaModel = self.BayesOpts.MetaModel
+ Discrepancy = self.BayesOpts.Discrepancy
+
+ # Find the number of sigma2 parameters
+ if Discrepancy.opt_sigma != 'B':
+ disc_bound_tuples = Discrepancy.ExpDesign.bound_tuples
+ disc_marginals = Discrepancy.ExpDesign.InputObj.Marginals
+ disc_prior_space = Discrepancy.ExpDesign.prior_space
+ n_sigma2 = len(disc_bound_tuples)
+ else:
+ n_sigma2 = -len(theta)
+ prior_dist = self.BayesOpts.engine.ExpDesign.prior_space
+ params_range = self.BayesOpts.engine.ExpDesign.bound_tuples
+ theta = theta if theta.ndim != 1 else theta.reshape((1, -1))
+ nsamples = theta.shape[0]
+ logprior = -np.inf*np.ones(nsamples)
+
+ for i in range(nsamples):
+ # Check if the sample is within the parameters' range
+ if self._check_ranges(theta[i], params_range):
+ # Check if all dists are uniform, if yes priors are equal.
+ if all(MetaModel.input_obj.Marginals[i].dist_type == 'uniform'
+ for i in range(MetaModel.n_params)):
+ logprior[i] = 0.0
+ else:
+ logprior[i] = np.log(
+ prior_dist.pdf(theta[i, :-n_sigma2].T)
+ )
+
+ # Check if bias term needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ if self._check_ranges(theta[i, -n_sigma2:],
+ disc_bound_tuples):
+ if all('unif' in disc_marginals[i].dist_type for i in
+ range(Discrepancy.ExpDesign.ndim)):
+ logprior[i] = 0.0
+ else:
+ logprior[i] += np.log(
+ disc_prior_space.pdf(theta[i, -n_sigma2:])
+ )
+
+ if nsamples == 1:
+ return logprior[0]
+ else:
+ return logprior
+
+ # -------------------------------------------------------------------------
+ def log_likelihood(self, theta):
+ """
+ Computes likelihood \\( p(\\mathcal{Y}|\\theta)\\) of the performance
+ of the (meta-)model in reproducing the observation data.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ log_like : array of shape (n_samples)
+ Log likelihood.
+
+ """
+
+ BayesOpts = self.BayesOpts
+ MetaModel = BayesOpts.MetaModel
+ Discrepancy = self.BayesOpts.Discrepancy
+
+ # Find the number of sigma2 parameters
+ if Discrepancy.opt_sigma != 'B':
+ disc_bound_tuples = Discrepancy.ExpDesign.bound_tuples
+ n_sigma2 = len(disc_bound_tuples)
+ else:
+ n_sigma2 = -len(theta)
+ # Check if bias term needs to be inferred
+ if Discrepancy.opt_sigma != 'B':
+ sigma2 = theta[:, -n_sigma2:]
+ theta = theta[:, :-n_sigma2]
+ else:
+ sigma2 = None
+ theta = theta if theta.ndim != 1 else theta.reshape((1, -1))
+
+ # Evaluate Model/MetaModel at theta
+ mean_pred, BayesOpts._std_pce_prior_pred = self.eval_model(theta)
+
+ # Surrogate model's error using RMSE of test data
+ surrError = MetaModel.rmse if hasattr(MetaModel, 'rmse') else None
+
+ # Likelihood
+ log_like = BayesOpts.normpdf(
+ mean_pred, self.observation, self.total_sigma2, sigma2,
+ std=surrError
+ )
+ return log_like
+
+ # -------------------------------------------------------------------------
+ def log_posterior(self, theta):
+ """
+ Computes the posterior likelihood \\(p(\\theta| \\mathcal{Y})\\) for
+ the given parameterset.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ log_like : array of shape (n_samples)
+ Log posterior likelihood.
+
+ """
+
+ nsamples = 1 if theta.ndim == 1 else theta.shape[0]
+
+ if nsamples == 1:
+ if self.log_prior(theta) == -np.inf:
+ return -np.inf
+ else:
+ # Compute log prior
+ log_prior = self.log_prior(theta)
+ # Compute log Likelihood
+ log_likelihood = self.log_likelihood(theta)
+
+ return log_prior + log_likelihood
+ else:
+ # Compute log prior
+ log_prior = self.log_prior(theta)
+
+ # Initialize log_likelihood
+ log_likelihood = -np.inf*np.ones(nsamples)
+
+ # find the indices for -inf sets
+ non_inf_idx = np.where(log_prior != -np.inf)[0]
+
+ # Compute loLikelihoods
+ if non_inf_idx.size != 0:
+ log_likelihood[non_inf_idx] = self.log_likelihood(
+ theta[non_inf_idx]
+ )
+
+ return log_prior + log_likelihood
+
+ # -------------------------------------------------------------------------
+ def eval_model(self, theta):
+ """
+ Evaluates the (meta-) model at the given theta.
+
+ Parameters
+ ----------
+ theta : array of shape (n_samples, n_params)
+ Parameter set, i.e. proposals of the MCMC chains.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean model prediction.
+ std_pred : dict
+ Std of model prediction.
+
+ """
+
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.MetaModel
+ Model = BayesObj.engine.Model
+
+ if BayesObj.emulator:
+ # Evaluate the MetaModel
+ mean_pred, std_pred = MetaModel.eval_metamodel(samples=theta)
+ else:
+ # Evaluate the origModel
+ mean_pred, std_pred = dict(), dict()
+
+ model_outs, _ = Model.run_model_parallel(
+ theta, prevRun_No=self.counter,
+ key_str='_MCMC', mp=False, verbose=False)
+
+ # Save outputs in respective dicts
+ for varIdx, var in enumerate(Model.Output.names):
+ mean_pred[var] = model_outs[var]
+ std_pred[var] = np.zeros((mean_pred[var].shape))
+
+ # Remove the folder
+ if Model.link_type.lower() != 'function':
+ shutil.rmtree(f"{Model.name}_MCMC_{self.counter+1}")
+
+ # Add one to the counter
+ self.counter += 1
+
+ if hasattr(self, 'error_MetaModel') and BayesObj.error_model:
+ meanPred, stdPred = self.error_MetaModel.eval_model_error(
+ BayesObj.BiasInputs, mean_pred
+ )
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ def train_error_model(self, sampler):
+ """
+ Trains an error model using a Gaussian Process Regression.
+
+ Parameters
+ ----------
+ sampler : obj
+ emcee sampler.
+
+ Returns
+ -------
+ error_MetaModel : obj
+ A error model.
+
+ """
+ BayesObj = self.BayesOpts
+ MetaModel = BayesObj.MetaModel
+
+ # Prepare the poster samples
+ try:
+ tau = sampler.get_autocorr_time(tol=0)
+ except emcee.autocorr.AutocorrError:
+ tau = 5
+
+ if all(np.isnan(tau)):
+ tau = 5
+
+ burnin = int(2*np.nanmax(tau))
+ thin = int(0.5*np.nanmin(tau)) if int(0.5*np.nanmin(tau)) != 0 else 1
+ finalsamples = sampler.get_chain(discard=burnin, flat=True, thin=thin)
+ posterior = finalsamples[:, :MetaModel.n_params]
+
+ # Select posterior mean as MAP
+ map_theta = posterior.mean(axis=0).reshape((1, MetaModel.n_params))
+ # MAP_theta = st.mode(Posterior_df,axis=0)[0]
+
+ # Evaluate the (meta-)model at the MAP
+ y_map, y_std_map = MetaModel.eval_metamodel(samples=map_theta)
+
+ # Train a GPR meta-model using MAP
+ error_MetaModel = MetaModel.create_model_error(
+ BayesObj.BiasInputs, y_map, name='Calib')
+
+ return error_MetaModel
+
+ # -------------------------------------------------------------------------
+ def gelman_rubin(self, chain, return_var=False):
+ """
+ The potential scale reduction factor (PSRF) defined by the variance
+ within one chain, W, with the variance between chains B.
+ Both variances are combined in a weighted sum to obtain an estimate of
+ the variance of a parameter \\( \\theta \\).The square root of the
+ ratio of this estimates variance to the within chain variance is called
+ the potential scale reduction.
+ For a well converged chain it should approach 1. Values greater than
+ 1.1 typically indicate that the chains have not yet fully converged.
+
+ Source: http://joergdietrich.github.io/emcee-convergence.html
+
+ https://github.com/jwalton3141/jwalton3141.github.io/blob/master/assets/posts/ESS/rwmh.py
+
+ Parameters
+ ----------
+ chain : array (n_walkers, n_steps, n_params)
+ The emcee ensamples.
+
+ Returns
+ -------
+ R_hat : float
+ The Gelman-Robin values.
+
+ """
+ m_chains, n_iters = chain.shape[:2]
+
+ # Calculate between-chain variance
+ θb = np.mean(chain, axis=1)
+ θbb = np.mean(θb, axis=0)
+ B_over_n = ((θbb - θb)**2).sum(axis=0)
+ B_over_n /= (m_chains - 1)
+
+ # Calculate within-chain variances
+ ssq = np.var(chain, axis=1, ddof=1)
+ W = np.mean(ssq, axis=0)
+
+ # (over) estimate of variance
+ var_θ = W * (n_iters - 1) / n_iters + B_over_n
+
+ if return_var:
+ return var_θ
+ else:
+ # The square root of the ratio of this estimates variance to the
+ # within chain variance
+ R_hat = np.sqrt(var_θ / W)
+ return R_hat
+
+ # -------------------------------------------------------------------------
+ def marginal_llk_emcee(self, sampler, nburn=None, logp=None, maxiter=1000):
+ """
+ The Bridge Sampling Estimator of the Marginal Likelihood based on
+ https://gist.github.com/junpenglao/4d2669d69ddfe1d788318264cdcf0583
+
+ Parameters
+ ----------
+ sampler : TYPE
+ MultiTrace, result of MCMC run.
+ nburn : int, optional
+ Number of burn-in step. The default is None.
+ logp : TYPE, optional
+ Model Log-probability function. The default is None.
+ maxiter : int, optional
+ Maximum number of iterations. The default is 1000.
+
+ Returns
+ -------
+ marg_llk : dict
+ Estimated Marginal log-Likelihood.
+
+ """
+ r0, tol1, tol2 = 0.5, 1e-10, 1e-4
+
+ if logp is None:
+ logp = sampler.log_prob_fn
+
+ # Split the samples into two parts
+ # Use the first 50% for fiting the proposal distribution
+ # and the second 50% in the iterative scheme.
+ if nburn is None:
+ mtrace = sampler.chain
+ else:
+ mtrace = sampler.chain[:, nburn:, :]
+
+ nchain, len_trace, nrofVars = mtrace.shape
+
+ N1_ = len_trace // 2
+ N1 = N1_*nchain
+ N2 = len_trace*nchain - N1
+
+ samples_4_fit = np.zeros((nrofVars, N1))
+ samples_4_iter = np.zeros((nrofVars, N2))
+ effective_n = np.zeros((nrofVars))
+
+ # matrix with already transformed samples
+ for var in range(nrofVars):
+
+ # for fitting the proposal
+ x = mtrace[:, :N1_, var]
+
+ samples_4_fit[var, :] = x.flatten()
+ # for the iterative scheme
+ x2 = mtrace[:, N1_:, var]
+ samples_4_iter[var, :] = x2.flatten()
+
+ # effective sample size of samples_4_iter, scalar
+ effective_n[var] = self._my_ESS(x2)
+
+ # median effective sample size (scalar)
+ neff = np.median(effective_n)
+
+ # get mean & covariance matrix and generate samples from proposal
+ m = np.mean(samples_4_fit, axis=1)
+ V = np.cov(samples_4_fit)
+ L = chol(V, lower=True)
+
+ # Draw N2 samples from the proposal distribution
+ gen_samples = m[:, None] + np.dot(
+ L, st.norm.rvs(0, 1, size=samples_4_iter.shape)
+ )
+
+ # Evaluate proposal distribution for posterior & generated samples
+ q12 = st.multivariate_normal.logpdf(samples_4_iter.T, m, V)
+ q22 = st.multivariate_normal.logpdf(gen_samples.T, m, V)
+
+ # Evaluate unnormalized posterior for posterior & generated samples
+ q11 = logp(samples_4_iter.T)
+ q21 = logp(gen_samples.T)
+
+ # Run iterative scheme:
+ tmp = self._iterative_scheme(
+ N1, N2, q11, q12, q21, q22, r0, neff, tol1, maxiter, 'r'
+ )
+ if ~np.isfinite(tmp['logml']):
+ warnings.warn(
+ "Logml could not be estimated within maxiter, rerunning with "
+ "adjusted starting value. Estimate might be more variable than"
+ " usual.")
+ # use geometric mean as starting value
+ r0_2 = np.sqrt(tmp['r_vals'][-2]*tmp['r_vals'][-1])
+ tmp = self._iterative_scheme(
+ q11, q12, q21, q22, r0_2, neff, tol2, maxiter, 'logml'
+ )
+
+ marg_llk = dict(
+ logml=tmp['logml'], niter=tmp['niter'], method="normal",
+ q11=q11, q12=q12, q21=q21, q22=q22
+ )
+ return marg_llk
+
+ # -------------------------------------------------------------------------
+ def _iterative_scheme(self, N1, N2, q11, q12, q21, q22, r0, neff, tol,
+ maxiter, criterion):
+ """
+ Iterative scheme as proposed in Meng and Wong (1996) to estimate the
+ marginal likelihood
+
+ """
+ l1 = q11 - q12
+ l2 = q21 - q22
+ # To increase numerical stability,
+ # subtracting the median of l1 from l1 & l2 later
+ lstar = np.median(l1)
+ s1 = neff/(neff + N2)
+ s2 = N2/(neff + N2)
+ r = r0
+ r_vals = [r]
+ logml = np.log(r) + lstar
+ criterion_val = 1 + tol
+
+ i = 0
+ while (i <= maxiter) & (criterion_val > tol):
+ rold = r
+ logmlold = logml
+ numi = np.exp(l2 - lstar)/(s1 * np.exp(l2 - lstar) + s2 * r)
+ deni = 1/(s1 * np.exp(l1 - lstar) + s2 * r)
+ if np.sum(~np.isfinite(numi))+np.sum(~np.isfinite(deni)) > 0:
+ warnings.warn(
+ """Infinite value in iterative scheme, returning NaN.
+ Try rerunning with more samples.""")
+ r = (N1/N2) * np.sum(numi)/np.sum(deni)
+ r_vals.append(r)
+ logml = np.log(r) + lstar
+ i += 1
+ if criterion == 'r':
+ criterion_val = np.abs((r - rold)/r)
+ elif criterion == 'logml':
+ criterion_val = np.abs((logml - logmlold)/logml)
+
+ if i >= maxiter:
+ return dict(logml=np.NaN, niter=i, r_vals=np.asarray(r_vals))
+ else:
+ return dict(logml=logml, niter=i)
+
+ # -------------------------------------------------------------------------
+ def _my_ESS(self, x):
+ """
+ Compute the effective sample size of estimand of interest.
+ Vectorised implementation.
+ https://github.com/jwalton3141/jwalton3141.github.io/blob/master/assets/posts/ESS/rwmh.py
+
+
+ Parameters
+ ----------
+ x : array of shape (n_walkers, n_steps)
+ MCMC Samples.
+
+ Returns
+ -------
+ int
+ Effective sample size.
+
+ """
+ m_chains, n_iters = x.shape
+
+ def variogram(t):
+ variogram = ((x[:, t:] - x[:, :(n_iters - t)])**2).sum()
+ variogram /= (m_chains * (n_iters - t))
+ return variogram
+
+ post_var = self.gelman_rubin(x, return_var=True)
+
+ t = 1
+ rho = np.ones(n_iters)
+ negative_autocorr = False
+
+ # Iterate until the sum of consecutive estimates of autocorrelation is
+ # negative
+ while not negative_autocorr and (t < n_iters):
+ rho[t] = 1 - variogram(t) / (2 * post_var)
+
+ if not t % 2:
+ negative_autocorr = sum(rho[t-1:t+1]) < 0
+
+ t += 1
+
+ return int(m_chains*n_iters / (1 + 2*rho[1:t].sum()))
+
+ # -------------------------------------------------------------------------
+ def _check_ranges(self, theta, ranges):
+ """
+ This function checks if theta lies in the given ranges.
+
+ Parameters
+ ----------
+ theta : array
+ Proposed parameter set.
+ ranges : nested list
+ List of the praremeter ranges.
+
+ Returns
+ -------
+ c : bool
+ If it lies in the given range, it return True else False.
+
+ """
+ c = True
+ # traverse in the list1
+ for i, bounds in enumerate(ranges):
+ x = theta[i]
+ # condition check
+ if x < bounds[0] or x > bounds[1]:
+ c = False
+ return c
+ return c
diff --git a/examples/model-comparison/bayesvalidrox/bayesvalidrox.mplstyle b/examples/model-comparison/bayesvalidrox/bayesvalidrox.mplstyle
new file mode 100644
index 0000000000000000000000000000000000000000..1f31c01f24597de0e0be741be4d3a706c4213a6c
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/bayesvalidrox.mplstyle
@@ -0,0 +1,16 @@
+figure.titlesize : 30
+axes.titlesize : 30
+axes.labelsize : 30
+axes.linewidth : 3
+axes.grid : True
+lines.linewidth : 3
+lines.markersize : 10
+xtick.labelsize : 30
+ytick.labelsize : 30
+legend.fontsize : 30
+font.family : serif
+font.serif : Arial
+font.size : 30
+text.usetex : True
+grid.linestyle : -
+figure.figsize : 24, 16
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__init__.py b/examples/model-comparison/bayesvalidrox/post_processing/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..81c9825420b6ed3f027fb3c141be8af05a89f695
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/post_processing/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .post_processing import PostProcessing
+
+__all__ = [
+ "PostProcessing"
+ ]
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c8590a242166b2e8d40de7ee2eece71980bd1571
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e84acd550fed2f7af8a071adf99001f44547bdf6
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..58a0eb24635d0b97a14d13708e616de6a0659976
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0924d8afac04d4fe82ebe791bc55a8ae48d7c117
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..94ffbf6f3da6b2b15cdf648a10ce9edb82d90834
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..312575d7655db85df423489051f494f3dce62692
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/post_processing/__pycache__/post_processing.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/post_processing/post_processing.py b/examples/model-comparison/bayesvalidrox/post_processing/post_processing.py
new file mode 100644
index 0000000000000000000000000000000000000000..6520a40f9f2393798f6b8abac026b9ed38fe33ca
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/post_processing/post_processing.py
@@ -0,0 +1,1338 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import math
+import os
+from itertools import combinations, cycle
+import pandas as pd
+import scipy.stats as stats
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error, r2_score
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+from matplotlib.offsetbox import AnchoredText
+from matplotlib.patches import Patch
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class PostProcessing:
+ """
+ This class provides many helper functions to post-process the trained
+ meta-model.
+
+ Attributes
+ ----------
+ MetaModel : obj
+ MetaModel object to do postprocessing on.
+ name : str
+ Type of the anaylsis. The default is `'calib'`. If a validation is
+ expected to be performed change this to `'valid'`.
+ """
+
+ def __init__(self, engine, name='calib'):
+ self.engine = engine
+ self.MetaModel = engine.MetaModel
+ self.ExpDesign = engine.ExpDesign
+ self.ModelObj = engine.Model
+ self.name = name
+
+ # -------------------------------------------------------------------------
+ def plot_moments(self, xlabel='Time [s]', plot_type=None):
+ """
+ Plots the moments in a pdf format in the directory
+ `Outputs_PostProcessing`.
+
+ Parameters
+ ----------
+ xlabel : str, optional
+ String to be displayed as x-label. The default is `'Time [s]'`.
+ plot_type : str, optional
+ Options: bar or line. The default is `None`.
+
+ Returns
+ -------
+ pce_means: dict
+ Mean of the model outputs.
+ pce_means: dict
+ Standard deviation of the model outputs.
+
+ """
+
+ bar_plot = True if plot_type == 'bar' else False
+ meta_model_type = self.MetaModel.meta_model_type
+ Model = self.ModelObj
+
+ # Read Monte-Carlo reference
+ self.mc_reference = Model.read_observation('mc_ref')
+
+ # Set the x values
+ x_values_orig = self.engine.ExpDesign.x_values
+
+ # Compute the moments with the PCEModel object
+ self.pce_means, self.pce_stds = self.compute_pce_moments()
+
+ # Get the variables
+ out_names = Model.Output.names
+
+ # Open a pdf for the plots
+ newpath = (f'Outputs_PostProcessing_{self.name}/')
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Plot the best fit line, set the linewidth (lw), color and
+ # transparency (alpha) of the line
+ for key in out_names:
+ fig, ax = plt.subplots(nrows=1, ncols=2)
+
+ # Extract mean and std
+ mean_data = self.pce_means[key]
+ std_data = self.pce_stds[key]
+
+ # Extract a list of x values
+ if type(x_values_orig) is dict:
+ x = x_values_orig[key]
+ else:
+ x = x_values_orig
+
+ # Plot: bar plot or line plot
+ if bar_plot:
+ ax[0].bar(list(map(str, x)), mean_data, color='b',
+ width=0.25)
+ ax[1].bar(list(map(str, x)), std_data, color='b',
+ width=0.25)
+ ax[0].legend(labels=[meta_model_type])
+ ax[1].legend(labels=[meta_model_type])
+ else:
+ ax[0].plot(x, mean_data, lw=3, color='k', marker='x',
+ label=meta_model_type)
+ ax[1].plot(x, std_data, lw=3, color='k', marker='x',
+ label=meta_model_type)
+
+ if self.mc_reference is not None:
+ if bar_plot:
+ ax[0].bar(list(map(str, x)), self.mc_reference['mean'],
+ color='r', width=0.25)
+ ax[1].bar(list(map(str, x)), self.mc_reference['std'],
+ color='r', width=0.25)
+ ax[0].legend(labels=[meta_model_type])
+ ax[1].legend(labels=[meta_model_type])
+ else:
+ ax[0].plot(x, self.mc_reference['mean'], lw=3, marker='x',
+ color='r', label='Ref.')
+ ax[1].plot(x, self.mc_reference['std'], lw=3, marker='x',
+ color='r', label='Ref.')
+
+ # Label the axes and provide a title
+ ax[0].set_xlabel(xlabel)
+ ax[1].set_xlabel(xlabel)
+ ax[0].set_ylabel(key)
+ ax[1].set_ylabel(key)
+
+ # Provide a title
+ ax[0].set_title('Mean of ' + key)
+ ax[1].set_title('Std of ' + key)
+
+ if not bar_plot:
+ ax[0].legend(loc='best')
+ ax[1].legend(loc='best')
+
+ plt.tight_layout()
+
+ # save the current figure
+ fig.savefig(
+ f'./{newpath}Mean_Std_PCE_{key}.pdf',
+ bbox_inches='tight'
+ )
+
+ return self.pce_means, self.pce_stds
+
+ # -------------------------------------------------------------------------
+ def valid_metamodel(self, n_samples=1, samples=None, model_out_dict=None,
+ x_axis='Time [s]'):
+ """
+ Evaluates and plots the meta model and the PCEModel outputs for the
+ given number of samples or the given samples.
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of samples to be evaluated. The default is 1.
+ samples : array of shape (n_samples, n_params), optional
+ Samples to be evaluated. The default is None.
+ model_out_dict: dict
+ The model runs using the samples provided.
+ x_axis : str, optional
+ Label of x axis. The default is `'Time [s]'`.
+
+ Returns
+ -------
+ None.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.ModelObj
+
+ if samples is None:
+ self.n_samples = n_samples
+ samples = self._get_sample()
+ else:
+ self.n_samples = samples.shape[0]
+
+ # Extract x_values
+ x_values = self.engine.ExpDesign.x_values
+
+ if model_out_dict is not None:
+ self.model_out_dict = model_out_dict
+ else:
+ self.model_out_dict = self._eval_model(samples, key_str='valid')
+ self.pce_out_mean, self.pce_out_std = MetaModel.eval_metamodel(samples)
+
+ try:
+ key = Model.Output.names[1]
+ except IndexError:
+ key = Model.Output.names[0]
+
+ n_obs = self.model_out_dict[key].shape[1]
+
+ if n_obs == 1:
+ self._plot_validation()
+ else:
+ self._plot_validation_multi(x_values=x_values, x_axis=x_axis)
+
+ # -------------------------------------------------------------------------
+ def check_accuracy(self, n_samples=None, samples=None, outputs=None):
+ """
+ Checks accuracy of the metamodel by computing the root mean square
+ error and validation error for all outputs.
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of samples. The default is None.
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets to be checked. The default is None.
+ outputs : dict, optional
+ Output dictionary with model outputs for all given output types in
+ `Model.Output.names`. The default is None.
+
+ Raises
+ ------
+ Exception
+ When neither n_samples nor samples are provided.
+
+ Returns
+ -------
+ rmse: dict
+ Root mean squared error for each output.
+ valid_error : dict
+ Validation error for each output.
+
+ """
+ MetaModel = self.MetaModel
+ Model = self.ModelObj
+
+ # Set the number of samples
+ if n_samples:
+ self.n_samples = n_samples
+ elif samples is not None:
+ self.n_samples = samples.shape[0]
+ else:
+ raise Exception("Please provide either samples or pass the number"
+ " of samples!")
+
+ # Generate random samples if necessary
+ Samples = self._get_sample() if samples is None else samples
+
+ # Run the original model with the generated samples
+ if outputs is None:
+ outputs = self._eval_model(Samples, key_str='validSet')
+
+ # Run the PCE model with the generated samples
+ pce_outputs, _ = MetaModel.eval_metamodel(samples=Samples)
+
+ self.rmse = {}
+ self.valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in Model.Output.names:
+ # Root mena square
+ self.rmse[key] = mean_squared_error(outputs[key], pce_outputs[key],
+ squared=False,
+ multioutput='raw_values')
+ # Validation error
+ self.valid_error[key] = (self.rmse[key]**2) / \
+ np.var(outputs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(self.rmse[key],
+ self.valid_error[key]))))
+ # Save error dicts in PCEModel object
+ self.MetaModel.rmse = self.rmse
+ self.MetaModel.valid_error = self.valid_error
+
+ return
+
+ # -------------------------------------------------------------------------
+ def plot_seq_design_diagnostics(self, ref_BME_KLD=None):
+ """
+ Plots the Bayesian Model Evidence (BME) and Kullback-Leibler divergence
+ (KLD) for the sequential design.
+
+ Parameters
+ ----------
+ ref_BME_KLD : array, optional
+ Reference BME and KLD . The default is `None`.
+
+ Returns
+ -------
+ None.
+
+ """
+ engine = self.engine
+ PCEModel = self.MetaModel
+ n_init_samples = engine.ExpDesign.n_init_samples
+ n_total_samples = engine.ExpDesign.X.shape[0]
+
+ newpath = f'Outputs_PostProcessing_{self.name}/seq_design_diagnostics/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ plotList = ['Modified LOO error', 'Validation error', 'KLD', 'BME',
+ 'RMSEMean', 'RMSEStd', 'Hellinger distance']
+ seqList = [engine.SeqModifiedLOO, engine.seqValidError,
+ engine.SeqKLD, engine.SeqBME, engine.seqRMSEMean,
+ engine.seqRMSEStd, engine.SeqDistHellinger]
+
+ markers = ('x', 'o', 'd', '*', '+')
+ colors = ('k', 'darkgreen', 'b', 'navy', 'darkred')
+
+ # Plot the evolution of the diagnostic criteria of the
+ # Sequential Experimental Design.
+ for plotidx, plot in enumerate(plotList):
+ fig, ax = plt.subplots()
+ seq_dict = seqList[plotidx]
+ name_util = list(seq_dict.keys())
+
+ if len(name_util) == 0:
+ continue
+
+ # Box plot when Replications have been detected.
+ if any(int(name.split("rep_", 1)[1]) > 1 for name in name_util):
+ # Extract the values from dict
+ sorted_seq_opt = {}
+ # Number of replications
+ n_reps = engine.ExpDesign.n_replication
+
+ # Get the list of utility function names
+ # Handle if only one UtilityFunction is provided
+ if not isinstance(engine.ExpDesign.util_func, list):
+ util_funcs = [engine.ExpDesign.util_func]
+ else:
+ util_funcs = engine.ExpDesign.util_func
+
+ for util in util_funcs:
+ sortedSeq = {}
+ # min number of runs available from reps
+ n_runs = min([seq_dict[f'{util}_rep_{i+1}'].shape[0]
+ for i in range(n_reps)])
+
+ for runIdx in range(n_runs):
+ values = []
+ for key in seq_dict.keys():
+ if util in key:
+ values.append(seq_dict[key][runIdx].mean())
+ sortedSeq['SeqItr_'+str(runIdx)] = np.array(values)
+ sorted_seq_opt[util] = sortedSeq
+
+ # BoxPlot
+ def draw_plot(data, labels, edge_color, fill_color, idx):
+ pos = labels - (idx-1)
+ bp = plt.boxplot(data, positions=pos, labels=labels,
+ patch_artist=True, sym='', widths=0.75)
+ elements = ['boxes', 'whiskers', 'fliers', 'means',
+ 'medians', 'caps']
+ for element in elements:
+ plt.setp(bp[element], color=edge_color[idx])
+
+ for patch in bp['boxes']:
+ patch.set(facecolor=fill_color[idx])
+
+ if engine.ExpDesign.n_new_samples != 1:
+ step1 = engine.ExpDesign.n_new_samples
+ step2 = 1
+ else:
+ step1 = 5
+ step2 = 5
+ edge_color = ['red', 'blue', 'green']
+ fill_color = ['tan', 'cyan', 'lightgreen']
+ plot_label = plot
+ # Plot for different Utility Functions
+ for idx, util in enumerate(util_funcs):
+ all_errors = np.empty((n_reps, 0))
+
+ for key in list(sorted_seq_opt[util].keys()):
+ errors = sorted_seq_opt.get(util, {}).get(key)[:, None]
+ all_errors = np.hstack((all_errors, errors))
+
+ # Special cases for BME and KLD
+ if plot == 'KLD' or plot == 'BME':
+ # BME convergence if refBME is provided
+ if ref_BME_KLD is not None:
+ if plot == 'BME':
+ refValue = ref_BME_KLD[0]
+ plot_label = r'BME/BME$^{Ref.}$'
+ if plot == 'KLD':
+ refValue = ref_BME_KLD[1]
+ plot_label = '$D_{KL}[p(\\theta|y_*),p(\\theta)]'\
+ ' / D_{KL}^{Ref.}[p(\\theta|y_*), '\
+ 'p(\\theta)]$'
+
+ # Difference between BME/KLD and the ref. values
+ all_errors = np.divide(all_errors,
+ np.full((all_errors.shape),
+ refValue))
+
+ # Plot baseline for zero, i.e. no difference
+ plt.axhline(y=1.0, xmin=0, xmax=1, c='green',
+ ls='--', lw=2)
+
+ # Plot each UtilFuncs
+ labels = np.arange(n_init_samples, n_total_samples+1, step1)
+ draw_plot(all_errors[:, ::step2], labels, edge_color,
+ fill_color, idx)
+
+ plt.xticks(labels, labels)
+ # Set the major and minor locators
+ ax.xaxis.set_major_locator(ticker.AutoLocator())
+ ax.xaxis.set_minor_locator(ticker.AutoMinorLocator())
+ ax.xaxis.grid(True, which='major', linestyle='-')
+ ax.xaxis.grid(True, which='minor', linestyle='--')
+
+ # Legend
+ legend_elements = []
+ for idx, util in enumerate(util_funcs):
+ legend_elements.append(Patch(facecolor=fill_color[idx],
+ edgecolor=edge_color[idx],
+ label=util))
+ plt.legend(handles=legend_elements[::-1], loc='best')
+
+ if plot != 'BME' and plot != 'KLD':
+ plt.yscale('log')
+ plt.autoscale(True)
+ plt.xlabel('\\# of training samples')
+ plt.ylabel(plot_label)
+ plt.title(plot)
+
+ # save the current figure
+ plot_name = plot.replace(' ', '_')
+ fig.savefig(
+ f'./{newpath}/seq_{plot_name}.pdf',
+ bbox_inches='tight'
+ )
+ # Destroy the current plot
+ plt.clf()
+ # Save arrays into files
+ f = open(f'./{newpath}/seq_{plot_name}.txt', 'w')
+ f.write(str(sorted_seq_opt))
+ f.close()
+ else:
+ for idx, name in enumerate(name_util):
+ seq_values = seq_dict[name]
+ if engine.ExpDesign.n_new_samples != 1:
+ step = engine.ExpDesign.n_new_samples
+ else:
+ step = 1
+ x_idx = np.arange(n_init_samples, n_total_samples+1, step)
+ if n_total_samples not in x_idx:
+ x_idx = np.hstack((x_idx, n_total_samples))
+
+ if plot == 'KLD' or plot == 'BME':
+ # BME convergence if refBME is provided
+ if ref_BME_KLD is not None:
+ if plot == 'BME':
+ refValue = ref_BME_KLD[0]
+ plot_label = r'BME/BME$^{Ref.}$'
+ if plot == 'KLD':
+ refValue = ref_BME_KLD[1]
+ plot_label = '$D_{KL}[p(\\theta|y_*),p(\\theta)]'\
+ ' / D_{KL}^{Ref.}[p(\\theta|y_*), '\
+ 'p(\\theta)]$'
+
+ # Difference between BME/KLD and the ref. values
+ values = np.divide(seq_values,
+ np.full((seq_values.shape),
+ refValue))
+
+ # Plot baseline for zero, i.e. no difference
+ plt.axhline(y=1.0, xmin=0, xmax=1, c='green',
+ ls='--', lw=2)
+
+ # Set the limits
+ plt.ylim([1e-1, 1e1])
+
+ # Create the plots
+ plt.semilogy(x_idx, values, marker=markers[idx],
+ color=colors[idx], ls='--', lw=2,
+ label=name.split("_rep", 1)[0])
+ else:
+ plot_label = plot
+
+ # Create the plots
+ plt.plot(x_idx, seq_values, marker=markers[idx],
+ color=colors[idx], ls='--', lw=2,
+ label=name.split("_rep", 1)[0])
+
+ else:
+ plot_label = plot
+ seq_values = np.nan_to_num(seq_values)
+
+ # Plot the error evolution for each output
+ plt.semilogy(x_idx, seq_values.mean(axis=1),
+ marker=markers[idx], ls='--', lw=2,
+ color=colors[idx],
+ label=name.split("_rep", 1)[0])
+
+ # Set the major and minor locators
+ ax.xaxis.set_major_locator(ticker.AutoLocator())
+ ax.xaxis.set_minor_locator(ticker.AutoMinorLocator())
+ ax.xaxis.grid(True, which='major', linestyle='-')
+ ax.xaxis.grid(True, which='minor', linestyle='--')
+
+ ax.tick_params(axis='both', which='major', direction='in',
+ width=3, length=10)
+ ax.tick_params(axis='both', which='minor', direction='in',
+ width=2, length=8)
+ plt.xlabel('Number of runs')
+ plt.ylabel(plot_label)
+ plt.title(plot)
+ plt.legend(frameon=True)
+
+ # save the current figure
+ plot_name = plot.replace(' ', '_')
+ fig.savefig(
+ f'./{newpath}/seq_{plot_name}.pdf',
+ bbox_inches='tight'
+ )
+ # Destroy the current plot
+ plt.clf()
+
+ # ---------------- Saving arrays into files ---------------
+ np.save(f'./{newpath}/seq_{plot_name}.npy', seq_values)
+
+ return
+
+ # -------------------------------------------------------------------------
+ def sobol_indices(self, xlabel='Time [s]', plot_type=None):
+ """
+ Provides Sobol indices as a sensitivity measure to infer the importance
+ of the input parameters. See Eq. 27 in [1] for more details. For the
+ case with Principal component analysis refer to [2].
+
+ [1] Global sensitivity analysis: A flexible and efficient framework
+ with an example from stochastic hydrogeology S. Oladyshkin, F.P.
+ de Barros, W. Nowak https://doi.org/10.1016/j.advwatres.2011.11.001
+
+ [2] Nagel, J.B., Rieckermann, J. and Sudret, B., 2020. Principal
+ component analysis and sparse polynomial chaos expansions for global
+ sensitivity analysis and model calibration: Application to urban
+ drainage simulation. Reliability Engineering & System Safety, 195,
+ p.106737.
+
+ Parameters
+ ----------
+ xlabel : str, optional
+ Label of the x-axis. The default is `'Time [s]'`.
+ plot_type : str, optional
+ Plot type. The default is `None`. This corresponds to line plot.
+ Bar chart can be selected by `bar`.
+
+ Returns
+ -------
+ sobol_cell: dict
+ Sobol indices.
+ total_sobol: dict
+ Total Sobol indices.
+
+ """
+ # Extract the necessary variables
+ PCEModel = self.MetaModel
+ basis_dict = PCEModel.basis_dict
+ coeffs_dict = PCEModel.coeffs_dict
+ n_params = PCEModel.n_params
+ max_order = np.max(PCEModel.pce_deg)
+ sobol_cell_b = {}
+ total_sobol_b = {}
+ cov_Z_p_q = np.zeros((n_params))
+
+ for b_i in range(PCEModel.n_bootstrap_itrs):
+
+ sobol_cell_, total_sobol_ = {}, {}
+
+ for output in self.ModelObj.Output.names:
+
+ n_meas_points = len(coeffs_dict[f'b_{b_i+1}'][output])
+
+ # Initialize the (cell) array containing the (total) Sobol indices.
+ sobol_array = dict.fromkeys(range(1, max_order+1), [])
+ sobol_cell_array = dict.fromkeys(range(1, max_order+1), [])
+
+ for i_order in range(1, max_order+1):
+ n_comb = math.comb(n_params, i_order)
+
+ sobol_cell_array[i_order] = np.zeros((n_comb, n_meas_points))
+
+ total_sobol_array = np.zeros((n_params, n_meas_points))
+
+ # Initialize the cell to store the names of the variables
+ TotalVariance = np.zeros((n_meas_points))
+ # Loop over all measurement points and calculate sobol indices
+ for pIdx in range(n_meas_points):
+
+ # Extract the basis indices (alpha) and coefficients
+ Basis = basis_dict[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+
+ try:
+ clf_poly = PCEModel.clf_poly[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+ PCECoeffs = clf_poly.coef_
+ except:
+ PCECoeffs = coeffs_dict[f'b_{b_i+1}'][output][f'y_{pIdx+1}']
+
+ # Compute total variance
+ TotalVariance[pIdx] = np.sum(np.square(PCECoeffs[1:]))
+
+ nzidx = np.where(PCECoeffs != 0)[0]
+ # Set all the Sobol indices equal to zero in the presence of a
+ # null output.
+ if len(nzidx) == 0:
+ # This is buggy.
+ for i_order in range(1, max_order+1):
+ sobol_cell_array[i_order][:, pIdx] = 0
+
+ # Otherwise compute them by summing well-chosen coefficients
+ else:
+ nz_basis = Basis[nzidx]
+ for i_order in range(1, max_order+1):
+ idx = np.where(np.sum(nz_basis > 0, axis=1) == i_order)
+ subbasis = nz_basis[idx]
+ Z = np.array(list(combinations(range(n_params), i_order)))
+
+ for q in range(Z.shape[0]):
+ Zq = Z[q]
+ subsubbasis = subbasis[:, Zq]
+ subidx = np.prod(subsubbasis, axis=1) > 0
+ sum_ind = nzidx[idx[0][subidx]]
+ if TotalVariance[pIdx] == 0.0:
+ sobol_cell_array[i_order][q, pIdx] = 0.0
+ else:
+ sobol = np.sum(np.square(PCECoeffs[sum_ind]))
+ sobol /= TotalVariance[pIdx]
+ sobol_cell_array[i_order][q, pIdx] = sobol
+
+ # Compute the TOTAL Sobol indices.
+ for ParIdx in range(n_params):
+ idx = nz_basis[:, ParIdx] > 0
+ sum_ind = nzidx[idx]
+
+ if TotalVariance[pIdx] == 0.0:
+ total_sobol_array[ParIdx, pIdx] = 0.0
+ else:
+ sobol = np.sum(np.square(PCECoeffs[sum_ind]))
+ sobol /= TotalVariance[pIdx]
+ total_sobol_array[ParIdx, pIdx] = sobol
+
+ # ----- if PCA selected: Compute covariance -----
+ if PCEModel.dim_red_method.lower() == 'pca':
+ # Extract the basis indices (alpha) and coefficients for
+ # next component
+ if pIdx < n_meas_points-1:
+ nextBasis = basis_dict[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+ if PCEModel.bootstrap_method != 'fast' or b_i == 0:
+ clf_poly = PCEModel.clf_poly[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+ nextPCECoeffs = clf_poly.coef_
+ else:
+ nextPCECoeffs = coeffs_dict[f'b_{b_i+1}'][output][f'y_{pIdx+2}']
+
+ # Choose the common non-zero basis
+ mask = (Basis[:, None] == nextBasis).all(-1).any(-1)
+ n_mask = (nextBasis[:, None] == Basis).all(-1).any(-1)
+
+ # Compute the covariance in Eq 17.
+ for ParIdx in range(n_params):
+ idx = (mask) & (Basis[:, ParIdx] > 0)
+ n_idx = (n_mask) & (nextBasis[:, ParIdx] > 0)
+ try:
+ cov_Z_p_q[ParIdx] += np.sum(np.dot(
+ PCECoeffs[idx], nextPCECoeffs[n_idx])
+ )
+ except:
+ pass
+
+ # Compute the sobol indices according to Ref. 2
+ if PCEModel.dim_red_method.lower() == 'pca':
+ n_c_points = self.engine.ExpDesign.Y[output].shape[1]
+ PCA = PCEModel.pca[f'b_{b_i+1}'][output]
+ compPCA = PCA.components_
+ nComp = compPCA.shape[0]
+ var_Z_p = PCA.explained_variance_
+
+ # Extract the sobol index of the components
+ for i_order in range(1, max_order+1):
+ n_comb = math.comb(n_params, i_order)
+ sobol_array[i_order] = np.zeros((n_comb, n_c_points))
+ Z = np.array(list(combinations(range(n_params), i_order)))
+
+ # Loop over parameters
+ for q in range(Z.shape[0]):
+ S_Z_i = sobol_cell_array[i_order][q]
+
+ for tIdx in range(n_c_points):
+ var_Y_t = np.var(
+ self.engine.ExpDesign.Y[output][:, tIdx])
+ if var_Y_t == 0.0:
+ term1, term2 = 0.0, 0.0
+ else:
+ # Eq. 17
+ term1 = 0.0
+ for i in range(nComp):
+ a = S_Z_i[i] * var_Z_p[i]
+ a *= compPCA[i, tIdx]**2
+ term1 += a
+
+ # TODO: Term 2
+ # term2 = 0.0
+ # for i in range(nComp-1):
+ # term2 += cov_Z_p_q[q] * compPCA[i, tIdx]
+ # term2 *= compPCA[i+1, tIdx]
+ # term2 *= 2
+
+ sobol_array[i_order][q, tIdx] = term1 #+ term2
+
+ # Devide over total output variance Eq. 18
+ sobol_array[i_order][q, tIdx] /= var_Y_t
+
+ # Compute the TOTAL Sobol indices.
+ total_sobol = np.zeros((n_params, n_c_points))
+ for ParIdx in range(n_params):
+ S_Z_i = total_sobol_array[ParIdx]
+
+ for tIdx in range(n_c_points):
+ var_Y_t = np.var(self.engine.ExpDesign.Y[output][:, tIdx])
+ if var_Y_t == 0.0:
+ term1, term2 = 0.0, 0.0
+ else:
+ term1 = 0
+ for i in range(nComp):
+ term1 += S_Z_i[i] * var_Z_p[i] * \
+ (compPCA[i, tIdx]**2)
+
+ # Term 2
+ term2 = 0
+ for i in range(nComp-1):
+ term2 += cov_Z_p_q[ParIdx] * compPCA[i, tIdx] \
+ * compPCA[i+1, tIdx]
+ term2 *= 2
+
+ total_sobol[ParIdx, tIdx] = term1 #+ term2
+
+ # Devide over total output variance Eq. 18
+ total_sobol[ParIdx, tIdx] /= var_Y_t
+
+ sobol_cell_[output] = sobol_array
+ total_sobol_[output] = total_sobol
+ else:
+ sobol_cell_[output] = sobol_cell_array
+ total_sobol_[output] = total_sobol_array
+
+ # Save for each bootsrtap iteration
+ sobol_cell_b[b_i] = sobol_cell_
+ total_sobol_b[b_i] = total_sobol_
+
+ # Average total sobol indices
+ total_sobol_all = {}
+ for i in sorted(total_sobol_b):
+ for k, v in total_sobol_b[i].items():
+ if k not in total_sobol_all:
+ total_sobol_all[k] = [None] * len(total_sobol_b)
+ total_sobol_all[k][i] = v
+
+ self.total_sobol = {}
+ for output in self.ModelObj.Output.names:
+ self.total_sobol[output] = np.mean(total_sobol_all[output], axis=0)
+
+ # ---------------- Plot -----------------------
+ par_names = self.engine.ExpDesign.par_names
+ x_values_orig = self.engine.ExpDesign.x_values
+
+ newpath = (f'Outputs_PostProcessing_{self.name}/')
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ fig = plt.figure()
+
+ for outIdx, output in enumerate(self.ModelObj.Output.names):
+
+ # Extract total Sobol indices
+ total_sobol = self.total_sobol[output]
+
+ # Compute quantiles
+ q_5 = np.quantile(total_sobol_all[output], q=0.05, axis=0)
+ q_97_5 = np.quantile(total_sobol_all[output], q=0.975, axis=0)
+
+ # Extract a list of x values
+ if type(x_values_orig) is dict:
+ x = x_values_orig[output]
+ else:
+ x = x_values_orig
+
+ if plot_type == 'bar':
+ ax = fig.add_axes([0, 0, 1, 1])
+ dict1 = {xlabel: x}
+ dict2 = {param: sobolIndices for param, sobolIndices
+ in zip(par_names, total_sobol)}
+
+ df = pd.DataFrame({**dict1, **dict2})
+ df.plot(x=xlabel, y=par_names, kind="bar", ax=ax, rot=0,
+ colormap='Dark2', yerr=q_97_5-q_5)
+ ax.set_ylabel('Total Sobol indices, $S^T$')
+
+ else:
+ for i, sobolIndices in enumerate(total_sobol):
+ plt.plot(x, sobolIndices, label=par_names[i],
+ marker='x', lw=2.5)
+ plt.fill_between(x, q_5[i], q_97_5[i], alpha=0.15)
+
+ plt.ylabel('Total Sobol indices, $S^T$')
+ plt.xlabel(xlabel)
+
+ plt.title(f'Sensitivity analysis of {output}')
+ if plot_type != 'bar':
+ plt.legend(loc='best', frameon=True)
+
+ # Save indices
+ np.savetxt(f'./{newpath}totalsobol_' +
+ output.replace('/', '_') + '.csv',
+ total_sobol.T, delimiter=',',
+ header=','.join(par_names), comments='')
+
+ # save the current figure
+ fig.savefig(
+ f'./{newpath}Sobol_indices_{output}.pdf',
+ bbox_inches='tight'
+ )
+
+ # Destroy the current plot
+ plt.clf()
+
+ return self.total_sobol
+
+ # -------------------------------------------------------------------------
+ def check_reg_quality(self, n_samples=1000, samples=None):
+ """
+ Checks the quality of the metamodel for single output models based on:
+ https://towardsdatascience.com/how-do-you-check-the-quality-of-your-regression-model-in-python-fa61759ff685
+
+
+ Parameters
+ ----------
+ n_samples : int, optional
+ Number of parameter sets to use for the check. The default is 1000.
+ samples : array of shape (n_samples, n_params), optional
+ Parameter sets to use for the check. The default is None.
+
+ Returns
+ -------
+ None.
+
+ """
+ MetaModel = self.MetaModel
+
+ if samples is None:
+ self.n_samples = n_samples
+ samples = self._get_sample()
+ else:
+ self.n_samples = samples.shape[0]
+
+ # Evaluate the original and the surrogate model
+ y_val = self._eval_model(samples, key_str='valid')
+ y_pce_val, _ = MetaModel.eval_metamodel(samples=samples)
+
+ # Open a pdf for the plots
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Fit the data(train the model)
+ for key in y_pce_val.keys():
+
+ y_pce_val_ = y_pce_val[key]
+ y_val_ = y_val[key]
+ residuals = y_val_ - y_pce_val_
+
+ # ------ Residuals vs. predicting variables ------
+ # Check the assumptions of linearity and independence
+ fig1 = plt.figure()
+ for i, par in enumerate(self.engine.ExpDesign.par_names):
+ plt.title(f"{key}: Residuals vs. {par}")
+ plt.scatter(
+ x=samples[:, i], y=residuals, color='blue', edgecolor='k')
+ plt.grid(True)
+ xmin, xmax = min(samples[:, i]), max(samples[:, i])
+ plt.hlines(y=0, xmin=xmin*0.9, xmax=xmax*1.1, color='red',
+ lw=3, linestyle='--')
+ plt.xlabel(par)
+ plt.ylabel('Residuals')
+ plt.show()
+
+ # save the current figure
+ fig1.savefig(f'./{newpath}/Residuals_vs_Par_{i+1}.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Fitted vs. residuals ------
+ # Check the assumptions of linearity and independence
+ fig2 = plt.figure()
+ plt.title(f"{key}: Residuals vs. fitted values")
+ plt.scatter(x=y_pce_val_, y=residuals, color='blue', edgecolor='k')
+ plt.grid(True)
+ xmin, xmax = min(y_val_), max(y_val_)
+ plt.hlines(y=0, xmin=xmin*0.9, xmax=xmax*1.1, color='red', lw=3,
+ linestyle='--')
+ plt.xlabel(key)
+ plt.ylabel('Residuals')
+ plt.show()
+
+ # save the current figure
+ fig2.savefig(f'./{newpath}/Fitted_vs_Residuals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Histogram of normalized residuals ------
+ fig3 = plt.figure()
+ resid_pearson = residuals / (max(residuals)-min(residuals))
+ plt.hist(resid_pearson, bins=20, edgecolor='k')
+ plt.ylabel('Count')
+ plt.xlabel('Normalized residuals')
+ plt.title(f"{key}: Histogram of normalized residuals")
+
+ # Normality (Shapiro-Wilk) test of the residuals
+ ax = plt.gca()
+ _, p = stats.shapiro(residuals)
+ if p < 0.01:
+ annText = "The residuals seem to come from a Gaussian Process."
+ else:
+ annText = "The normality assumption may not hold."
+ at = AnchoredText(annText, prop=dict(size=30), frameon=True,
+ loc='upper left')
+ at.patch.set_boxstyle("round,pad=0.,rounding_size=0.2")
+ ax.add_artist(at)
+
+ plt.show()
+
+ # save the current figure
+ fig3.savefig(f'./{newpath}/Hist_NormResiduals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # ------ Q-Q plot of the normalized residuals ------
+ plt.figure()
+ stats.probplot(residuals[:, 0], plot=plt)
+ plt.xticks()
+ plt.yticks()
+ plt.xlabel("Theoretical quantiles")
+ plt.ylabel("Sample quantiles")
+ plt.title(f"{key}: Q-Q plot of normalized residuals")
+ plt.grid(True)
+ plt.show()
+
+ # save the current figure
+ plt.savefig(f'./{newpath}/QQPlot_NormResiduals.pdf',
+ bbox_inches='tight')
+ # Destroy the current plot
+ plt.clf()
+
+ # -------------------------------------------------------------------------
+ def eval_pce_model_3d(self):
+
+ self.n_samples = 1000
+
+ PCEModel = self.MetaModel
+ Model = self.ModelObj
+ n_samples = self.n_samples
+
+ # Create 3D-Grid
+ # TODO: Make it general
+ x = np.linspace(-5, 10, n_samples)
+ y = np.linspace(0, 15, n_samples)
+
+ X, Y = np.meshgrid(x, y)
+ PCE_Z = np.zeros((self.n_samples, self.n_samples))
+ Model_Z = np.zeros((self.n_samples, self.n_samples))
+
+ for idxMesh in range(self.n_samples):
+ sample_mesh = np.vstack((X[:, idxMesh], Y[:, idxMesh])).T
+
+ univ_p_val = PCEModel.univ_basis_vals(sample_mesh)
+
+ for Outkey, ValuesDict in PCEModel.coeffs_dict.items():
+
+ pce_out_mean = np.zeros((len(sample_mesh), len(ValuesDict)))
+ pce_out_std = np.zeros((len(sample_mesh), len(ValuesDict)))
+ model_outs = np.zeros((len(sample_mesh), len(ValuesDict)))
+
+ for Inkey, InIdxValues in ValuesDict.items():
+ idx = int(Inkey.split('_')[1]) - 1
+ basis_deg_ind = PCEModel.basis_dict[Outkey][Inkey]
+ clf_poly = PCEModel.clf_poly[Outkey][Inkey]
+
+ PSI_Val = PCEModel.create_psi(basis_deg_ind, univ_p_val)
+
+ # Perdiction with error bar
+ y_mean, y_std = clf_poly.predict(PSI_Val, return_std=True)
+
+ pce_out_mean[:, idx] = y_mean
+ pce_out_std[:, idx] = y_std
+
+ # Model evaluation
+ model_out_dict, _ = Model.run_model_parallel(sample_mesh,
+ key_str='Valid3D')
+ model_outs[:, idx] = model_out_dict[Outkey].T
+
+ PCE_Z[:, idxMesh] = y_mean
+ Model_Z[:, idxMesh] = model_outs[:, 0]
+
+ # ---------------- 3D plot for PCEModel -----------------------
+ fig_PCE = plt.figure()
+ ax = plt.axes(projection='3d')
+ ax.plot_surface(X, Y, PCE_Z, rstride=1, cstride=1,
+ cmap='viridis', edgecolor='none')
+ ax.set_title('PCEModel')
+ ax.set_xlabel('$x_1$')
+ ax.set_ylabel('$x_2$')
+ ax.set_zlabel('$f(x_1,x_2)$')
+
+ plt.grid()
+ plt.show()
+
+ # Saving the figure
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # save the figure to file
+ fig_PCE.savefig(f'./{newpath}/3DPlot_PCEModel.pdf',
+ bbox_inches='tight')
+ plt.close(fig_PCE)
+
+ # ---------------- 3D plot for Model -----------------------
+ fig_Model = plt.figure()
+ ax = plt.axes(projection='3d')
+ ax.plot_surface(X, Y, PCE_Z, rstride=1, cstride=1,
+ cmap='viridis', edgecolor='none')
+ ax.set_title('Model')
+ ax.set_xlabel('$x_1$')
+ ax.set_ylabel('$x_2$')
+ ax.set_zlabel('$f(x_1,x_2)$')
+
+ plt.grid()
+ plt.show()
+
+ # Save the figure
+ fig_Model.savefig(f'./{newpath}/3DPlot_Model.pdf',
+ bbox_inches='tight')
+ plt.close(fig_Model)
+
+ return
+
+ # -------------------------------------------------------------------------
+ def compute_pce_moments(self):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moments (mean) of outpust.
+ pce_means: dict
+ The first moments (mean) of outpust.
+
+ """
+
+ MetaModel = self.MetaModel
+ outputs = self.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = np.sqrt(np.dot(pce_var,
+ PCA.components_**2))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ # Print a report table
+ print("\n>>>>> Moments of {} <<<<<".format(output))
+ print("\nIndex | Mean | Std. deviation")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(pce_means[output],
+ pce_stds[output]))))
+ print('-'*40)
+
+ return pce_means, pce_stds
+
+ # -------------------------------------------------------------------------
+ def _get_sample(self, n_samples=None):
+ """
+ Generates random samples taken from the input parameter space.
+
+ Returns
+ -------
+ samples : array of shape (n_samples, n_params)
+ Generated samples.
+
+ """
+ if n_samples is None:
+ n_samples = self.n_samples
+ self.samples = self.ExpDesign.generate_samples(
+ n_samples,
+ sampling_method='random')
+ return self.samples
+
+ # -------------------------------------------------------------------------
+ def _eval_model(self, samples=None, key_str='Valid'):
+ """
+ Evaluates Forward Model for the given number of self.samples or given
+ samples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate the model at. The default is None.
+ key_str : str, optional
+ Key string pass to the model. The default is 'Valid'.
+
+ Returns
+ -------
+ model_outs : dict
+ Dictionary of results.
+
+ """
+ Model = self.ModelObj
+
+ if samples is None:
+ samples = self._get_sample()
+ self.samples = samples
+ else:
+ self.n_samples = len(samples)
+
+ model_outs, _ = Model.run_model_parallel(samples, key_str=key_str)
+
+ return model_outs
+
+ # -------------------------------------------------------------------------
+ def _plot_validation(self):
+ """
+ Plots outputs for visual comparison of metamodel outputs with that of
+ the (full) original model.
+
+ Returns
+ -------
+ None.
+
+ """
+ PCEModel = self.MetaModel
+
+ # get the samples
+ x_val = self.samples
+ y_pce_val = self.pce_out_mean
+ y_val = self.model_out_dict
+
+ # Open a pdf for the plots
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ fig = plt.figure()
+ # Fit the data(train the model)
+ for key in y_pce_val.keys():
+
+ y_pce_val_ = y_pce_val[key]
+ y_val_ = y_val[key]
+
+ regression_model = LinearRegression()
+ regression_model.fit(y_pce_val_, y_val_)
+
+ # Predict
+ x_new = np.linspace(np.min(y_pce_val_), np.max(y_val_), 100)
+ y_predicted = regression_model.predict(x_new[:, np.newaxis])
+
+ plt.scatter(y_pce_val_, y_val_, color='gold', linewidth=2)
+ plt.plot(x_new, y_predicted, color='k')
+
+ # Calculate the adjusted R_squared and RMSE
+ # the total number of explanatory variables in the model
+ # (not including the constant term)
+ length_list = []
+ for key, value in PCEModel.coeffs_dict['b_1'][key].items():
+ length_list.append(len(value))
+ n_predictors = min(length_list)
+ n_samples = x_val.shape[0]
+
+ R2 = r2_score(y_pce_val_, y_val_)
+ AdjR2 = 1 - (1 - R2) * (n_samples - 1) / \
+ (n_samples - n_predictors - 1)
+ rmse = mean_squared_error(y_pce_val_, y_val_, squared=False)
+
+ plt.annotate(f'RMSE = {rmse:.3f}\n Adjusted $R^2$ = {AdjR2:.3f}',
+ xy=(0.05, 0.85), xycoords='axes fraction')
+
+ plt.ylabel("Original Model")
+ plt.xlabel("PCE Model")
+ plt.grid()
+ plt.show()
+
+ # save the current figure
+ plot_name = key.replace(' ', '_')
+ fig.savefig(f'./{newpath}/Model_vs_PCEModel_{plot_name}.pdf',
+ bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ # -------------------------------------------------------------------------
+ def _plot_validation_multi(self, x_values=[], x_axis="x [m]"):
+ """
+ Plots outputs for visual comparison of metamodel outputs with that of
+ the (full) multioutput original model
+
+ Parameters
+ ----------
+ x_values : list or array, optional
+ List of x values. The default is [].
+ x_axis : str, optional
+ Label of the x axis. The default is "x [m]".
+
+ Returns
+ -------
+ None.
+
+ """
+ Model = self.ModelObj
+
+ newpath = f'Outputs_PostProcessing_{self.name}/'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # List of markers and colors
+ color = cycle((['b', 'g', 'r', 'y', 'k']))
+ marker = cycle(('x', 'd', '+', 'o', '*'))
+
+ fig = plt.figure()
+ # Plot the model vs PCE model
+ for keyIdx, key in enumerate(Model.Output.names):
+
+ y_pce_val = self.pce_out_mean[key]
+ y_pce_val_std = self.pce_out_std[key]
+ y_val = self.model_out_dict[key]
+ try:
+ x = self.model_out_dict['x_values'][key]
+ except (TypeError, IndexError):
+ x = x_values
+
+ for idx in range(y_val.shape[0]):
+ Color = next(color)
+ Marker = next(marker)
+
+ plt.plot(x, y_val[idx], color=Color, marker=Marker,
+ label='$Y_{%s}^M$'%(idx+1))
+
+ plt.plot(x, y_pce_val[idx], color=Color, marker=Marker,
+ linestyle='--',
+ label='$Y_{%s}^{PCE}$'%(idx+1))
+ plt.fill_between(x, y_pce_val[idx]-1.96*y_pce_val_std[idx],
+ y_pce_val[idx]+1.96*y_pce_val_std[idx],
+ color=Color, alpha=0.15)
+
+ # Calculate the RMSE
+ rmse = mean_squared_error(y_pce_val, y_val, squared=False)
+ R2 = r2_score(y_pce_val[idx].reshape(-1, 1),
+ y_val[idx].reshape(-1, 1))
+
+ plt.annotate(f'RMSE = {rmse:.3f}\n $R^2$ = {R2:.3f}',
+ xy=(0.85, 0.1), xycoords='axes fraction')
+
+ plt.ylabel(key)
+ plt.xlabel(x_axis)
+ plt.legend(loc='best')
+ plt.grid()
+
+ # save the current figure
+ plot_name = key.replace(' ', '_')
+ fig.savefig(f'./{newpath}/Model_vs_PCEModel_{plot_name}.pdf',
+ bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+
+ # Zip the subdirectories
+ Model.zip_subdirs(f'{Model.name}valid', f'{Model.name}valid_')
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__init__.py b/examples/model-comparison/bayesvalidrox/pylink/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4bd81739faf43956324b30f6d8e5365b29d55677
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/pylink/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .pylink import PyLinkForwardModel
+
+__all__ = [
+ "PyLinkForwardModel"
+ ]
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5b7c1b3926506fb279b856f55ca6120df31b8888
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1947ad354716d0293953761f0d35193f706cedc1
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0bbb522855ad250ad55bca46123c0f5023076291
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b6ae7c14b35b60388e38fcbd3af64d04771a947c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0f0f850911d36dce30d3a9e6f59478e5216044c8
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..746c82eb52be2e437c61bd201433f9d38b8ab177
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/pylink/__pycache__/pylink.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/pylink/pylink.py b/examples/model-comparison/bayesvalidrox/pylink/pylink.py
new file mode 100644
index 0000000000000000000000000000000000000000..227a51ab38cd834e7e85f6193d83563c7ed3437a
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/pylink/pylink.py
@@ -0,0 +1,803 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Calls to the model and evaluations
+"""
+
+from dataclasses import dataclass
+
+import os
+import shutil
+import h5py
+import numpy as np
+import time
+import zipfile
+import pandas as pd
+import multiprocessing
+from functools import partial
+import tqdm
+
+#from multiprocessing import get_context
+from multiprocess import get_context
+
+
+
+def within_range(out, minout, maxout):
+ """
+ Checks if all the values in out lie between minout and maxout
+
+ Parameters
+ ----------
+ out : array or list
+ Data to check against range
+ minout : int
+ Lower bound of the range
+ maxout : int
+ Upper bound of the range
+
+ Returns
+ -------
+ inside : bool
+ True if all values in out are in the specified range
+
+ """
+ try:
+ out = np.array(out)
+ except:
+ raise AttributeError('The given values should be a 1D array, but are not')
+ if out.ndim != 1:
+ raise AttributeError('The given values should be a 1D array, but are not')
+
+ if minout > maxout:
+ raise ValueError('The lower and upper bounds do not form a valid range, they might be switched')
+
+ inside = False
+ if (out > minout).all() and (out < maxout).all():
+ inside = True
+ return inside
+
+
+class PyLinkForwardModel(object):
+ """
+ A forward model binder
+
+ This calss serves as a code wrapper. This wrapper allows the execution of
+ a third-party software/solver within the scope of BayesValidRox.
+
+ Attributes
+ ----------
+ link_type : str
+ The type of the wrapper. The default is `'pylink'`. This runs the
+ third-party software or an executable using a shell command with given
+ input files.
+ Second option is `'function'` which assumed that model can be run using
+ a function written separately in a Python script.
+ name : str
+ Name of the model.
+ py_file : str
+ Python file name without `.py` extension to be run for the `'function'`
+ wrapper. Note that the name of the python file and that of the function
+ must be simillar. This function must recieve the parameters in an array
+ of shape `(n_samples, n_params)` and returns a dictionary with the
+ x_values and output arrays for given output names.
+ func_args : dict
+ Additional arguments for the python file. The default is `{}`.
+ shell_command : str
+ Shell command to be executed for the `'pylink'` wrapper.
+ input_file : str or list
+ The input file to be passed to the `'pylink'` wrapper.
+ input_template : str or list
+ A template input file to be passed to the `'pylink'` wrapper. This file
+ must be a copy of `input_file` with `<Xi>` place holder for the input
+ parameters defined using `inputs` class, with i being the number of
+ parameter. The file name ending should include `.tpl` before the actual
+ extension of the input file, for example, `params.tpl.input`.
+ aux_file : str or list
+ The list of auxiliary files needed for the `'pylink'` wrapper.
+ exe_path : str
+ Execution path if you wish to run the model for the `'pylink'` wrapper
+ in another directory. The default is `None`, which corresponds to the
+ currecnt working directory.
+ output_file_names : list of str
+ List of the name of the model output text files for the `'pylink'`
+ wrapper.
+ output_names : list of str
+ List of the model outputs to be used for the analysis.
+ output_parser : str
+ Name of the model parser file (without `.py` extension) that recieves
+ the `output_file_names` and returns a 2d-array with the first row being
+ the x_values, e.g. x coordinates or time and the rest of raws pass the
+ simulation output for each model output defined in `output_names`. Note
+ that again here the name of the file and that of the function must be
+ the same.
+ multi_process: bool
+ Whether the model runs to be executed in parallel for the `'pylink'`
+ wrapper. The default is `True`.
+ n_cpus: int
+ The number of cpus to be used for the parallel model execution for the
+ `'pylink'` wrapper. The default is `None`, which corresponds to all
+ available cpus.
+ meas_file : str
+ The name of the measurement text-based file. This file must contain
+ x_values as the first column and one column for each model output. The
+ default is `None`. Only needed for the Bayesian Inference.
+ meas_file_valid : str
+ The name of the measurement text-based file for the validation. The
+ default is `None`. Only needed for the validation with Bayesian
+ Inference.
+ mc_ref_file : str
+ The name of the text file for the Monte-Carlo reference (mean and
+ standard deviation) values. It must contain `x_values` as the first
+ column, `mean` as the second column and `std` as the third. It can be
+ used to compare the estimated moments using meta-model in the post-
+ processing step. This is only available for one output.
+ obs_dict : dict
+ A dictionary containing the measurement text-based file. It must
+ contain `x_values` as the first item and one item for each model output
+ . The default is `{}`. Only needed for the Bayesian Inference.
+ obs_dict_valid : dict
+ A dictionary containing the validation measurement text-based file. It
+ must contain `x_values` as the first item and one item for each model
+ output. The default is `{}`.
+ mc_ref_dict : dict
+ A dictionary containing the Monte-Carlo reference (mean and standard
+ deviation) values. It must contain `x_values` as the first item and
+ `mean` as the second item and `std` as the third. The default is `{}`.
+ This is only available for one output.
+ """
+
+ # Nested class
+ @dataclass
+ class OutputData(object):
+ parser: str = ""
+ names: list = None
+ file_names: list = None
+
+ def __init__(self, link_type='pylink', name=None, py_file=None,
+ func_args={}, shell_command='', input_file=None,
+ input_template=None, aux_file=None, exe_path='',
+ output_file_names=[], output_names=[], output_parser='',
+ multi_process=True, n_cpus=None, meas_file=None,
+ meas_file_valid=None, mc_ref_file=None, obs_dict={},
+ obs_dict_valid={}, mc_ref_dict={}):
+ self.link_type = link_type
+ self.name = name
+ self.shell_command = shell_command
+ self.py_file = py_file
+ self.func_args = func_args
+ self.input_file = input_file
+ self.input_template = input_template
+ self.aux_file = aux_file
+ self.exe_path = exe_path
+ self.multi_process = multi_process
+ self.n_cpus = n_cpus
+ self.Output = self.OutputData(
+ parser=output_parser,
+ names=output_names,
+ file_names=output_file_names,
+ )
+ self.n_outputs = len(self.Output.names)
+ self.meas_file = meas_file
+ self.meas_file_valid = meas_file_valid
+ self.mc_ref_file = mc_ref_file
+ self.observations = obs_dict
+ self.observations_valid = obs_dict_valid
+ self.mc_reference = mc_ref_dict
+
+ # -------------------------------------------------------------------------
+ def read_observation(self, case='calib'):
+ """
+ Reads/prepare the observation/measurement data for
+ calibration.
+
+ Parameters
+ ----------
+ case : str
+ The type of observation to read in. Can be either 'calib',
+ 'valid' or 'mc_ref'
+
+ Returns
+ -------
+ DataFrame
+ A dataframe with the calibration data.
+
+ """
+ # TOOD: check that what is read in/transformed matches the expected form of data/reference
+ if case.lower() == 'calib':
+ if isinstance(self.observations, dict) and bool(self.observations):
+ self.observations = pd.DataFrame.from_dict(self.observations)
+ elif self.meas_file is not None:
+ file_path = os.path.join(os.getcwd(), self.meas_file)
+ self.observations = pd.read_csv(file_path, delimiter=',')
+ elif isinstance(self.observations, pd.DataFrame):
+ self.observations = self.observations
+ else:
+ raise Exception("Please provide the observation data as a "
+ "dictionary via observations attribute or pass"
+ " the csv-file path to MeasurementFile "
+ "attribute")
+ # Compute the number of observation
+ self.n_obs = self.observations[self.Output.names].notnull().sum().values.sum()
+ return self.observations
+
+ elif case.lower() == 'valid':
+ if isinstance(self.observations_valid, dict) and \
+ bool(self.observations_valid):
+ self.observations_valid = pd.DataFrame.from_dict(self.observations_valid)
+ elif self.meas_file_valid is not None:
+ file_path = os.path.join(os.getcwd(), self.meas_file_valid)
+ self.observations_valid = pd.read_csv(file_path, delimiter=',')
+ elif isinstance(self.observations_valid, pd.DataFrame):
+ self.observations_valid = self.observations_valid
+ else:
+ raise Exception("Please provide the observation data as a "
+ "dictionary via observations attribute or pass"
+ " the csv-file path to MeasurementFile "
+ "attribute")
+ # Compute the number of observation
+ self.n_obs_valid = self.observations_valid[self.Output.names].notnull().sum().values.sum()
+ return self.observations_valid
+
+ elif case.lower() == 'mc_ref':
+ if self.mc_ref_file is None and \
+ isinstance(self.mc_reference, pd.DataFrame):
+ return self.mc_reference
+ elif isinstance(self.mc_reference, dict) and bool(self.mc_reference):
+ self.mc_reference = pd.DataFrame.from_dict(self.mc_reference)
+ elif self.mc_ref_file is not None:
+ file_path = os.path.join(os.getcwd(), self.mc_ref_file)
+ self.mc_reference = pd.read_csv(file_path, delimiter=',')
+ else:
+ self.mc_reference = None
+ return self.mc_reference
+
+
+ # -------------------------------------------------------------------------
+ def read_output(self):
+ """
+ Reads the the parser output file and returns it as an
+ executable function. It is required when the models returns the
+ simulation outputs in csv files.
+
+ Returns
+ -------
+ Output : func
+ Output parser function.
+
+ """
+ output_func_name = self.Output.parser
+
+ output_func = getattr(__import__(output_func_name), output_func_name)
+
+ file_names = []
+ for File in self.Output.file_names:
+ file_names.append(os.path.join(self.exe_path, File))
+ try:
+ output = output_func(self.name, file_names)
+ except TypeError:
+ output = output_func(file_names)
+ return output
+
+ # -------------------------------------------------------------------------
+ def update_input_params(self, new_input_file, param_set):
+ """
+ Finds this pattern with <X1> in the new_input_file and replace it with
+ the new value from the array param_sets.
+
+ Parameters
+ ----------
+ new_input_file : list
+ List of the input files with the adapted names.
+ param_set : array of shape (n_params)
+ Parameter set.
+
+ Returns
+ -------
+ None.
+
+ """
+ NofPa = param_set.shape[0]
+ text_to_search_list = [f'<X{i+1}>' for i in range(NofPa)]
+
+ for filename in new_input_file:
+ # Read in the file
+ with open(filename, 'r') as file:
+ filedata = file.read()
+
+ # Replace the target string
+ for text_to_search, params in zip(text_to_search_list, param_set):
+ filedata = filedata.replace(text_to_search, f'{params:0.4e}')
+
+ # Write the file out again
+ with open(filename, 'w') as file:
+ file.write(filedata)
+
+ # -------------------------------------------------------------------------
+ def run_command(self, command, output_file_names):
+ """
+ Runs the execution command given by the user to run the given model.
+ It checks if the output files have been generated. If yes, the jobe is
+ done and it extracts and returns the requested output(s). Otherwise,
+ it executes the command again.
+
+ Parameters
+ ----------
+ command : str
+ The shell command to be executed.
+ output_file_names : list
+ Name of the output file names.
+
+ Returns
+ -------
+ simulation_outputs : array of shape (n_obs, n_outputs)
+ Simulation outputs.
+
+ """
+
+ # Check if simulation is finished
+ while True:
+ time.sleep(3)
+ files = os.listdir(".")
+ if all(elem in files for elem in output_file_names):
+ break
+ else:
+ # Run command
+ Process = os.system(f'./../{command}')
+ if Process != 0:
+ print('\nMessage 1:')
+ print(f'\tIf the value of \'{Process}\' is a non-zero value'
+ ', then compilation problems occur \n' % Process)
+ os.chdir("..")
+
+ # Read the output
+ simulation_outputs = self.read_output()
+
+ return simulation_outputs
+
+ # -------------------------------------------------------------------------
+ def run_forwardmodel(self, xx):
+ """
+ This function creates subdirectory for the current run and copies the
+ necessary files to this directory and renames them. Next, it executes
+ the given command.
+
+ Parameters
+ ----------
+ xx : tuple
+ A tuple including parameter set, simulation number and key string.
+
+ Returns
+ -------
+ output : array of shape (n_outputs+1, n_obs)
+ An array passed by the output paraser containing the x_values as
+ the first row and the simulations results stored in the the rest of
+ the array.
+
+ """
+ c_points, run_no, key_str = xx
+
+ # Handle if only one imput file is provided
+ if not isinstance(self.input_template, list):
+ self.input_template = [self.input_template]
+ if not isinstance(self.input_file, list):
+ self.input_file = [self.input_file]
+
+ new_input_file = []
+ # Loop over the InputTemplates:
+ for in_temp in self.input_template:
+ if '/' in in_temp:
+ in_temp = in_temp.split('/')[-1]
+ new_input_file.append(in_temp.split('.tpl')[0] + key_str +
+ f"_{run_no+1}" + in_temp.split('.tpl')[1])
+
+ # Create directories
+ newpath = self.name + key_str + f'_{run_no+1}'
+ if not os.path.exists(newpath):
+ os.makedirs(newpath)
+
+ # Copy the necessary files to the directories
+ print(self.input_template)
+ for in_temp in self.input_template:
+ # Input file(s) of the model
+ shutil.copy2(in_temp, newpath)
+ # Auxiliary file
+ if self.aux_file is not None:
+ shutil.copy2(self.aux_file, newpath) # Auxiliary file
+
+ # Rename the Inputfile and/or auxiliary file
+ os.chdir(newpath)
+ for input_tem, input_file in zip(self.input_template, new_input_file):
+ if '/' in input_tem:
+ input_tem = input_tem.split('/')[-1]
+ os.rename(input_tem, input_file)
+
+ # Update the parametrs in Input file
+ self.update_input_params(new_input_file, c_points)
+
+ # Update the user defined command and the execution path
+ try:
+ new_command = self.shell_command.replace(self.input_file[0],
+ new_input_file[0])
+ new_command = new_command.replace(self.input_file[1],
+ new_input_file[1])
+ except:
+ new_command = self.shell_command.replace(self.input_file[0],
+ new_input_file[0])
+ # Set the exe path if not provided
+ if not bool(self.exe_path):
+ self.exe_path = os.getcwd()
+
+ # Run the model
+ print(new_command)
+ output = self.run_command(new_command, self.Output.file_names)
+
+ return output
+
+ # -------------------------------------------------------------------------
+ def run_model_parallel(self, c_points, prevRun_No=0, key_str='',
+ mp=True, verbose=True):
+ """
+ Runs model simulations. If mp is true (default), then the simulations
+ are started in parallel.
+
+ Parameters
+ ----------
+ c_points : array of shape (n_samples, n_params)
+ Collocation points (training set).
+ prevRun_No : int, optional
+ Previous run number, in case the sequential design is selected.
+ The default is `0`.
+ key_str : str, optional
+ A descriptive string for validation runs. The default is `''`.
+ mp : bool, optional
+ Multiprocessing. The default is `True`.
+ verbose: bool, optional
+ Verbosity. The default is `True`.
+
+ Returns
+ -------
+ all_outputs : dict
+ A dictionary with x values (time step or point id) and all outputs.
+ Each key contains an array of the shape `(n_samples, n_obs)`.
+ new_c_points : array
+ Updated collocation points (training set). If a simulation does not
+ executed successfully, the parameter set is removed.
+
+ """
+
+ # Initilization
+ n_c_points = len(c_points)
+ all_outputs = {}
+
+ # If the link type is UM-Bridge, then no parallel needs to be started from here
+ if self.link_type.lower() == 'umbridge':
+ import umbridge
+ if not hasattr(self, 'x_values'):
+ raise AttributeError('For model type `umbridge` the attribute `x_values` needs to be set for the model!')
+ # Init model
+ #model = umbridge.HTTPModel('http://localhost:4242', 'forward')
+ self.model = umbridge.HTTPModel(self.host, 'forward') # TODO: is this always forward?
+ Function = self.uMBridge_model
+
+ # Extract the function
+ if self.link_type.lower() == 'function':
+ # Prepare the function
+ Function = getattr(__import__(self.py_file), self.py_file)
+ # ---------------------------------------------------------------
+ # -------------- Multiprocessing with Pool Class ----------------
+ # ---------------------------------------------------------------
+ # Start a pool with the number of CPUs
+ if self.n_cpus is None:
+ n_cpus = multiprocessing.cpu_count()
+ else:
+ n_cpus = self.n_cpus
+
+ # Run forward model
+ if n_c_points == 1 or not mp:
+ if n_c_points== 1:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ group_results = Function(c_points, **self.func_args)
+ else:
+ group_results = self.run_forwardmodel(
+ (c_points[0], prevRun_No, key_str)
+ )
+ else:
+ for i in range(c_points.shape[0]):
+ if i == 0:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ group_results = Function(np.array([c_points[0]]), **self.func_args)
+ else:
+ group_results = self.run_forwardmodel(
+ (c_points[0], prevRun_No, key_str)
+ )
+ for key in group_results:
+ if key != 'x_values':
+ group_results[key] = [group_results[key]]
+ else:
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ res = Function(np.array([c_points[i]]), **self.func_args)
+ else:
+ res = self.run_forwardmodel(
+ (c_points[i], prevRun_No, key_str)
+ )
+ for key in res:
+ if key != 'x_values':
+ group_results[key].append(res[key])
+
+ for key in group_results:
+ if key != 'x_values':
+ group_results[key]= np.array(group_results[key])
+
+ elif self.multi_process or mp:
+ with get_context('spawn').Pool(n_cpus) as p:
+ #with multiprocessing.Pool(n_cpus) as p:
+
+ if self.link_type.lower() == 'function' or self.link_type.lower() == 'umbridge':
+ imap_var = p.imap(partial(Function, **self.func_args),
+ c_points[:, np.newaxis])
+ else:
+ args = zip(c_points,
+ [prevRun_No+i for i in range(n_c_points)],
+ [key_str]*n_c_points)
+ imap_var = p.imap(self.run_forwardmodel, args)
+
+ if verbose:
+ desc = f'Running forward model {key_str}'
+ group_results = list(tqdm.tqdm(imap_var, total=n_c_points,
+ desc=desc))
+ else:
+ group_results = list(imap_var)
+
+ # Check for NaN
+ for var_i, var in enumerate(self.Output.names):
+ # If results are given as one dictionary
+ if isinstance(group_results, dict):
+ Outputs = np.asarray(group_results[var])
+ # If results are given as list of dictionaries
+ elif isinstance(group_results, list):
+ Outputs = np.asarray([item[var] for item in group_results],
+ dtype=np.float64)
+ NaN_idx = np.unique(np.argwhere(np.isnan(Outputs))[:, 0])
+ new_c_points = np.delete(c_points, NaN_idx, axis=0)
+ all_outputs[var] = np.atleast_2d(
+ np.delete(Outputs, NaN_idx, axis=0)
+ )
+
+ # Print the collocation points whose simulations crashed
+ if len(NaN_idx) != 0:
+ print('\n')
+ print('*'*20)
+ print("\nThe following parameter sets have been removed:\n",
+ c_points[NaN_idx])
+ print("\n")
+ print('*'*20)
+
+ # Save time steps or x-values
+ if isinstance(group_results, dict):
+ all_outputs["x_values"] = group_results["x_values"]
+ elif any(isinstance(i, dict) for i in group_results):
+ all_outputs["x_values"] = group_results[0]["x_values"]
+
+ # Store simulations in a hdf5 file
+ self._store_simulations(
+ c_points, all_outputs, NaN_idx, key_str, prevRun_No
+ )
+
+ return all_outputs, new_c_points
+
+ def uMBridge_model(self, params):
+ """
+ Function that calls a UMBridge model and transforms its output into the
+ shape expected for the surrogate.
+
+ Parameters
+ ----------
+ params : 2d np.array, shape (#samples, #params)
+ The parameter values for which the model is run.
+
+ Returns
+ -------
+ dict
+ The transformed model outputs.
+
+ """
+ # Run the model
+ #out = np.array(model(np.ndarray.tolist(params), {'level':0}))
+ out = np.array(self.model(np.ndarray.tolist(params), self.modelparams))
+
+ # Sort into dict
+ out_dict = {}
+ cnt = 0
+ for key in self.Output.names:
+ # # If needed resort into single-value outputs
+ # if self.output_type == 'single-valued':
+ # if out.shape[1]>1: # TODO: this doesn't fully seem correct??
+ # for i in range(out[:,key]): # TODO: this doesn't fully seem correct??
+ # new_key = key+str(i)
+ # if new_key not in self.Output.names:
+ # self.Output.names.append(new_key)
+ # if i == 0:
+ # self.Ouptut.names.remove(key)
+ # out_dict[new_key] = out[:,cnt,i] # TODO: not sure about this, need to test
+ # else:
+ # out_dict[key] = out[:,cnt]
+ #
+ #
+ # else:
+ out_dict[key] = out[:,cnt]
+ cnt += 1
+
+
+ ## TODO: how to deal with the x-values?
+ #if self.output_type == 'single-valued':
+ # out_dict['x_values'] = [0]
+ #else:
+ # out_dict['x_values'] = np.arange(0,out[:,0].shape[0],1)
+ out_dict['x_values'] = self.x_values
+
+ #return {'T1':out[:,0], 'T2':out[:,1], 'H1':out[:,2], 'H2':out[:,3],
+ # 'x_values':[0]}
+ return out_dict
+
+ # -------------------------------------------------------------------------
+ def _store_simulations(self, c_points, all_outputs, NaN_idx, key_str,
+ prevRun_No):
+ """
+
+
+ Parameters
+ ----------
+ c_points : TYPE
+ DESCRIPTION.
+ all_outputs : TYPE
+ DESCRIPTION.
+ NaN_idx : TYPE
+ DESCRIPTION.
+ key_str : TYPE
+ DESCRIPTION.
+ prevRun_No : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ None.
+
+ """
+
+ # Create hdf5 metadata
+ if key_str == '':
+ hdf5file = f'ExpDesign_{self.name}.hdf5'
+ else:
+ hdf5file = f'ValidSet_{self.name}.hdf5'
+ hdf5_exist = os.path.exists(hdf5file)
+ file = h5py.File(hdf5file, 'a')
+
+ # ---------- Save time steps or x-values ----------
+ if not hdf5_exist:
+ if type(all_outputs["x_values"]) is dict:
+ grp_x_values = file.create_group("x_values/")
+ for varIdx, var in enumerate(self.Output.names):
+ grp_x_values.create_dataset(
+ var, data=all_outputs["x_values"][var]
+ )
+ else:
+ file.create_dataset("x_values", data=all_outputs["x_values"])
+
+ # ---------- Save outputs ----------
+ for varIdx, var in enumerate(self.Output.names):
+ if not hdf5_exist:
+ grpY = file.create_group("EDY/"+var)
+ else:
+ grpY = file.get("EDY/"+var)
+
+ if prevRun_No == 0 and key_str == '':
+ grpY.create_dataset(f'init_{key_str}', data=all_outputs[var])
+ else:
+ try:
+ oldEDY = np.array(file[f'EDY/{var}/adaptive_{key_str}'])
+ del file[f'EDY/{var}/adaptive_{key_str}']
+ data = np.vstack((oldEDY, all_outputs[var]))
+ except KeyError:
+ data = all_outputs[var]
+ grpY.create_dataset('adaptive_'+key_str, data=data)
+
+ if prevRun_No == 0 and key_str == '':
+ grpY.create_dataset(f"New_init_{key_str}",
+ data=all_outputs[var])
+ else:
+ try:
+ name = f'EDY/{var}/New_adaptive_{key_str}'
+ oldEDY = np.array(file[name])
+ del file[f'EDY/{var}/New_adaptive_{key_str}']
+ data = np.vstack((oldEDY, all_outputs[var]))
+ except KeyError:
+ data = all_outputs[var]
+ grpY.create_dataset(f'New_adaptive_{key_str}', data=data)
+
+ # ---------- Save CollocationPoints ----------
+ new_c_points = np.delete(c_points, NaN_idx, axis=0)
+ grpX = file.create_group("EDX") if not hdf5_exist else file.get("EDX")
+ if prevRun_No == 0 and key_str == '':
+ grpX.create_dataset("init_"+key_str, data=c_points)
+ if len(NaN_idx) != 0:
+ grpX.create_dataset("New_init_"+key_str, data=new_c_points)
+
+ else:
+ try:
+ name = f'EDX/adaptive_{key_str}'
+ oldCollocationPoints = np.array(file[name])
+ del file[f'EDX/adaptive_{key_str}']
+ data = np.vstack((oldCollocationPoints, new_c_points))
+ except KeyError:
+ data = new_c_points
+ grpX.create_dataset('adaptive_'+key_str, data=data)
+
+ if len(NaN_idx) != 0:
+ try:
+ name = f'EDX/New_adaptive_{key_str}'
+ oldCollocationPoints = np.array(file[name])
+ del file[f'EDX/New_adaptive_{key_str}']
+ data = np.vstack((oldCollocationPoints, new_c_points))
+ except KeyError:
+ data = new_c_points
+ grpX.create_dataset('New_adaptive_'+key_str, data=data)
+
+ # Close h5py file
+ file.close()
+
+ # -------------------------------------------------------------------------
+ def zip_subdirs(self, dir_name, key):
+ """
+ Zips all the files containing the key(word).
+
+ Parameters
+ ----------
+ dir_name : str
+ Directory name.
+ key : str
+ Keyword to search for.
+
+ Returns
+ -------
+ None.
+
+ """
+ # setup file paths variable
+ dir_list = []
+ file_paths = []
+
+ # Read all directory, subdirectories and file lists
+ dir_path = os.getcwd()
+
+ for root, directories, files in os.walk(dir_path):
+ for directory in directories:
+ # Create the full filepath by using os module.
+ if key in directory:
+ folderPath = os.path.join(dir_path, directory)
+ dir_list.append(folderPath)
+
+ # Loop over the identified directories to store the file paths
+ for direct_name in dir_list:
+ for root, directories, files in os.walk(direct_name):
+ for filename in files:
+ # Create the full filepath by using os module.
+ filePath = os.path.join(root, filename)
+ file_paths.append('.'+filePath.split(dir_path)[1])
+
+ # writing files to a zipfile
+ if len(file_paths) != 0:
+ zip_file = zipfile.ZipFile(dir_name+'.zip', 'w')
+ with zip_file:
+ # writing each file one by one
+ for file in file_paths:
+ zip_file.write(file)
+
+ file_paths = [path for path in os.listdir('.') if key in path]
+
+ for path in file_paths:
+ shutil.rmtree(path)
+
+ print("\n")
+ print(f'{dir_name}.zip has been created successfully!\n')
+
+ return
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__init__.py b/examples/model-comparison/bayesvalidrox/surrogate_models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..70bfb20f570464c2907a0a4128f4ed99b6c13736
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/__init__.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+
+from .surrogate_models import MetaModel
+
+__all__ = [
+ "MetaModel"
+ ]
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8c10c82287a57ba1e3b4dd428962e57cdfbc5c58
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4b73e63a3096fbc9afc41bae35a3fcc1d7851166
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f1a3fcc2eed66172304cd27ab5fe111ca0198bf5
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/__init__.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2854217e56fecb2456011a91a984951fed9cbcbb
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/adaptPlot.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ffab8b4f67e52a3128aa8740301f958a0d72c502
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5d2b8aa7d552e4f4afc87a25caf36ab034f31486
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..82737a42dd7351d06b703b3da838031ba95979da
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/apoly_construction.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..92d0cc0e7a0a07123fdfbc2c777d1b9281a43344
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..56cfc2006febc94fdf721712929918499cd46491
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..063355b16a397fb5fd89d38daa0d3ca5a8506766
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/bayes_linear.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0a98675fd4b4b1172c75b81b88112c0e39880261
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/engine.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cbfe4d97e8e83ebc276e45ba6e84f514784f1d0b
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d789b4515f89e02f006dafd2c9d85b8e7bea110
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..27878248cbdaa2abf0af9d51d061aa6e2db86f43
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/eval_rec_rule.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc01f0f01b8bc70d438a3317b87d304883456f15
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a392bbec968c4728540bfbd2470d6cd4efb67faa
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e7e0226dc1e28c09b9cd09a610599007b2267e3c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..076580dd7fa9e11559ef202903d44e00e25b8a26
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exp_designs_.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..af7431ce432969f095d1c07f429b8129cb5a2def
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9a435919a7163073324fbaf96093a1aeb0b6387b
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..98964152a4ea29f85f061ea6ec7daa3df7487230
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/exploration.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..16a383b994853cdb226f7b7fb291cdbef789e1f7
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ffc6990e1d99e9ad0a3c41c689e313d7f680d8e
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c3f24cf9cb753b59f9e226f828a7c3598ac65a9a
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/glexindex.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2ff32cef7757bc4a20197f768e3cbfe819f9d428
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/input_space.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ed2b0f6f101965fb42fe059ec79e8084a4b3a9bf
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..615d2fe8b67cb014059c4fab5f06344ca0878adf
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b0c91ff3182adb1245aa7f20656e8e94336a438c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/inputs.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a1e26394365e683dbec6da95fbe223ebcee10ef4
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/loss_function.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ebbfad9c34b1f9e6c819ea7cf7852af65591444
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b2b3f8f40181637b13ee1156ffa8a03c1ffce8b3
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..bd6eb8fcd459fd95ff1b85514f996344b6e4880c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/meta_model_engine.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..543416416ef052c2402c2e9a97976dc4aab22866
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9e909177d8f6cb2a1722e871da4bc92ebfea493c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f1ec9479869fec42f363e7f46e07dba2f1c6be5
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/orthogonal_matching_pursuit.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..97ed55acc1e800a138ddf489ccea709a8b28f634
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4ab80e19a7070efd8d180d21c3b492cc333d73dd
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0843cdf8bd820d9cccfdcde3a1193f3f416ccc10
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_ard.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..30079dce4bc04802324f720381b872c1a2f64018
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f089d06b16a5698dee5f706d007dfa13916a42c
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7a3bbc05003eab405d52a934e83053ae50080d25
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/reg_fast_laplace.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..af35ff42f1ea1e8ee19d476f7b51c19199513cde
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/sequential_design.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b856dfef2c2658af8ecc6b1ab85b99c499a39705
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-310.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..569685a009aea33d28959c005328402624029ef7
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f44a774b4165e6ff769d8db2f2c13c4dd0cbe8b3
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/surrogate_models.cpython-39.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a6d61c979e164a0fd590b3f925c2d1ac68adb4fc
Binary files /dev/null and b/examples/model-comparison/bayesvalidrox/surrogate_models/__pycache__/wrapper.cpython-311.pyc differ
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/adaptPlot.py b/examples/model-comparison/bayesvalidrox/surrogate_models/adaptPlot.py
new file mode 100644
index 0000000000000000000000000000000000000000..102f0373c1086ba4420ada2fb2fc723b78bbd53f
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/adaptPlot.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Aug 13 13:46:24 2020
+
+@author: farid
+"""
+import os
+from sklearn.metrics import mean_squared_error, r2_score
+from itertools import cycle
+from matplotlib.backends.backend_pdf import PdfPages
+import matplotlib.pyplot as plt
+
+
+def adaptPlot(PCEModel, Y_Val, Y_PC_Val, Y_PC_Val_std, x_values=[],
+ plotED=False, SaveFig=True):
+
+ NrofSamples = PCEModel.ExpDesign.n_new_samples
+ initNSamples = PCEModel.ExpDesign.n_init_samples
+ itrNr = 1 + (PCEModel.ExpDesign.X.shape[0] - initNSamples)//NrofSamples
+
+ oldEDY = PCEModel.ExpDesign.Y
+
+ if SaveFig:
+ newpath = 'adaptivePlots'
+ os.makedirs(newpath, exist_ok=True)
+
+ # create a PdfPages object
+ pdf = PdfPages(f'./{newpath}/Model_vs_PCEModel_itr_{itrNr}.pdf')
+
+ # List of markers and colors
+ color = cycle((['b', 'g', 'r', 'y', 'k']))
+ marker = cycle(('x', 'd', '+', 'o', '*'))
+
+ OutNames = list(Y_Val.keys())
+ x_axis = 'Time [s]'
+
+ if len(OutNames) == 1:
+ OutNames.insert(0, x_axis)
+ try:
+ x_values = Y_Val['x_values']
+ except KeyError:
+ x_values = x_values
+
+ fig = plt.figure(figsize=(24, 16))
+
+ # Plot the model vs PCE model
+ for keyIdx, key in enumerate(PCEModel.ModelObj.Output.names):
+ Y_PC_Val_ = Y_PC_Val[key]
+ Y_PC_Val_std_ = Y_PC_Val_std[key]
+ Y_Val_ = Y_Val[key]
+ if Y_Val_.ndim == 1:
+ Y_Val_ = Y_Val_.reshape(1, -1)
+ old_EDY = oldEDY[key]
+ if isinstance(x_values, dict):
+ x = x_values[key]
+ else:
+ x = x_values
+
+ for idx, y in enumerate(Y_Val_):
+ Color = next(color)
+ Marker = next(marker)
+
+ plt.plot(
+ x, y, color=Color, marker=Marker,
+ lw=2.0, label='$Y_{%s}^{M}$'%(idx+itrNr)
+ )
+
+ plt.plot(
+ x, Y_PC_Val_[idx], color=Color, marker=Marker,
+ lw=2.0, linestyle='--', label='$Y_{%s}^{PCE}$'%(idx+itrNr)
+ )
+ plt.fill_between(
+ x, Y_PC_Val_[idx]-1.96*Y_PC_Val_std_[idx],
+ Y_PC_Val_[idx]+1.96*Y_PC_Val_std_[idx], color=Color,
+ alpha=0.15
+ )
+
+ if plotED:
+ for output in old_EDY:
+ plt.plot(x, output, color='grey', alpha=0.1)
+
+ # Calculate the RMSE
+ RMSE = mean_squared_error(Y_PC_Val_, Y_Val_, squared=False)
+ R2 = r2_score(Y_PC_Val_.reshape(-1, 1), Y_Val_.reshape(-1, 1))
+
+ plt.ylabel(key)
+ plt.xlabel(x_axis)
+ plt.title(key)
+
+ ax = fig.axes[0]
+ ax.legend(loc='best', frameon=True)
+ fig.canvas.draw()
+ ax.text(0.65, 0.85,
+ f'RMSE = {round(RMSE, 3)}\n$R^2$ = {round(R2, 3)}',
+ transform=ax.transAxes, color='black',
+ bbox=dict(facecolor='none',
+ edgecolor='black',
+ boxstyle='round,pad=1')
+ )
+ plt.grid()
+
+ if SaveFig:
+ # save the current figure
+ pdf.savefig(fig, bbox_inches='tight')
+
+ # Destroy the current plot
+ plt.clf()
+ pdf.close()
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/apoly_construction.py b/examples/model-comparison/bayesvalidrox/surrogate_models/apoly_construction.py
new file mode 100644
index 0000000000000000000000000000000000000000..40830fe8aaa94248df4828c0c49bd4d23e755abd
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/apoly_construction.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import numpy as np
+
+
+def apoly_construction(Data, degree):
+ """
+ Construction of Data-driven Orthonormal Polynomial Basis
+ Author: Dr.-Ing. habil. Sergey Oladyshkin
+ Department of Stochastic Simulation and Safety Research for Hydrosystems
+ Institute for Modelling Hydraulic and Environmental Systems
+ Universitaet Stuttgart, Pfaffenwaldring 5a, 70569 Stuttgart
+ E-mail: Sergey.Oladyshkin@iws.uni-stuttgart.de
+ http://www.iws-ls3.uni-stuttgart.de
+ The current script is based on definition of arbitrary polynomial chaos
+ expansion (aPC), which is presented in the following manuscript:
+ Oladyshkin, S. and W. Nowak. Data-driven uncertainty quantification using
+ the arbitrary polynomial chaos expansion. Reliability Engineering & System
+ Safety, Elsevier, V. 106, P. 179-190, 2012.
+ DOI: 10.1016/j.ress.2012.05.002.
+
+ Parameters
+ ----------
+ Data : array
+ Raw data.
+ degree : int
+ Maximum polynomial degree.
+
+ Returns
+ -------
+ Polynomial : array
+ The coefficients of the univariate orthonormal polynomials.
+
+ """
+ if Data.ndim !=1:
+ raise AttributeError('Data should be a 1D array')
+
+ # Initialization
+ dd = degree + 1
+ nsamples = len(Data)
+
+ # Forward linear transformation (Avoiding numerical issues)
+ MeanOfData = np.mean(Data)
+ Data = Data/MeanOfData
+
+ # Compute raw moments of input data
+ raw_moments = [np.sum(np.power(Data, p))/nsamples for p in range(2*dd+2)]
+
+ # Main Loop for Polynomial with degree up to dd
+ PolyCoeff_NonNorm = np.empty((0, 1))
+ Polynomial = np.zeros((dd+1, dd+1))
+
+ for degree in range(dd+1):
+ Mm = np.zeros((degree+1, degree+1))
+ Vc = np.zeros((degree+1))
+
+ # Define Moments Matrix Mm
+ for i in range(degree+1):
+ for j in range(degree+1):
+ if (i < degree):
+ Mm[i, j] = raw_moments[i+j]
+
+ elif (i == degree) and (j == degree):
+ Mm[i, j] = 1
+
+ # Numerical Optimization for Matrix Solver
+ Mm[i] = Mm[i] / max(abs(Mm[i]))
+
+ # Defenition of Right Hand side ortogonality conditions: Vc
+ for i in range(degree+1):
+ Vc[i] = 1 if i == degree else 0
+
+ # Solution: Coefficients of Non-Normal Orthogonal Polynomial: Vp Eq.(4)
+ try:
+ Vp = np.linalg.solve(Mm, Vc)
+ except:
+ inv_Mm = np.linalg.pinv(Mm)
+ Vp = np.dot(inv_Mm, Vc.T)
+
+ if degree == 0:
+ PolyCoeff_NonNorm = np.append(PolyCoeff_NonNorm, Vp)
+
+ if degree != 0:
+ if degree == 1:
+ zero = [0]
+ else:
+ zero = np.zeros((degree, 1))
+ PolyCoeff_NonNorm = np.hstack((PolyCoeff_NonNorm, zero))
+
+ PolyCoeff_NonNorm = np.vstack((PolyCoeff_NonNorm, Vp))
+
+ if 100*abs(sum(abs(np.dot(Mm, Vp)) - abs(Vc))) > 0.5:
+ print('\n---> Attention: Computational Error too high !')
+ print('\n---> Problem: Convergence of Linear Solver')
+
+ # Original Numerical Normalization of Coefficients with Norm and
+ # orthonormal Basis computation Matrix Storrage
+ # Note: Polynomial(i,j) correspont to coefficient number "j-1"
+ # of polynomial degree "i-1"
+ P_norm = 0
+ for i in range(nsamples):
+ Poly = 0
+ for k in range(degree+1):
+ if degree == 0:
+ Poly += PolyCoeff_NonNorm[k] * (Data[i]**k)
+ else:
+ Poly += PolyCoeff_NonNorm[degree, k] * (Data[i]**k)
+
+ P_norm += Poly**2 / nsamples
+
+ P_norm = np.sqrt(P_norm)
+
+ for k in range(degree+1):
+ if degree == 0:
+ Polynomial[degree, k] = PolyCoeff_NonNorm[k]/P_norm
+ else:
+ Polynomial[degree, k] = PolyCoeff_NonNorm[degree, k]/P_norm
+
+ # Backward linear transformation to the real data space
+ Data *= MeanOfData
+ for k in range(len(Polynomial)):
+ Polynomial[:, k] = Polynomial[:, k] / (MeanOfData**(k))
+
+ return Polynomial
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/bayes_linear.py b/examples/model-comparison/bayesvalidrox/surrogate_models/bayes_linear.py
new file mode 100644
index 0000000000000000000000000000000000000000..3bd827ac0ecc5b3a38116b21767e8a8799593b24
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/bayes_linear.py
@@ -0,0 +1,523 @@
+import numpy as np
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+from sklearn.utils import check_X_y, check_array, as_float_array
+from sklearn.utils.validation import check_is_fitted
+from scipy.linalg import svd
+import warnings
+from sklearn.preprocessing import normalize as f_normalize
+
+
+
+class BayesianLinearRegression(RegressorMixin,LinearModel):
+ '''
+ Superclass for Empirical Bayes and Variational Bayes implementations of
+ Bayesian Linear Regression Model
+ '''
+ def __init__(self, n_iter, tol, fit_intercept,copy_X, verbose):
+ self.n_iter = n_iter
+ self.fit_intercept = fit_intercept
+ self.copy_X = copy_X
+ self.verbose = verbose
+ self.tol = tol
+
+
+ def _check_convergence(self, mu, mu_old):
+ '''
+ Checks convergence of algorithm using changes in mean of posterior
+ distribution of weights
+ '''
+ return np.sum(abs(mu-mu_old)>self.tol) == 0
+
+
+ def _center_data(self,X,y):
+ ''' Centers data'''
+ X = as_float_array(X,copy = self.copy_X)
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype = X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X,axis = 0)
+ y_mean = np.average(y,axis = 0)
+ X -= X_mean
+ y = y - y_mean
+ else:
+ X_mean = np.zeros(X.shape[1],dtype = X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X,y, X_mean, y_mean, X_std
+
+
+ def predict_dist(self,X):
+ '''
+ Calculates mean and variance of predictive distribution for each data
+ point of test set.(Note predictive distribution for each data point is
+ Gaussian, therefore it is uniquely determined by mean and variance)
+
+ Parameters
+ ----------
+ x: array-like of size (n_test_samples, n_features)
+ Set of features for which corresponding responses should be predicted
+
+ Returns
+ -------
+ :list of two numpy arrays [mu_pred, var_pred]
+
+ mu_pred : numpy array of size (n_test_samples,)
+ Mean of predictive distribution
+
+ var_pred: numpy array of size (n_test_samples,)
+ Variance of predictive distribution
+ '''
+ # Note check_array and check_is_fitted are done within self._decision_function(X)
+ mu_pred = self._decision_function(X)
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ return [mu_pred,var_pred]
+
+
+
+
+class EBLinearRegression(BayesianLinearRegression):
+ '''
+ Bayesian Regression with type II maximum likelihood (Empirical Bayes)
+
+ Parameters:
+ -----------
+ n_iter: int, optional (DEFAULT = 300)
+ Maximum number of iterations
+
+ tol: float, optional (DEFAULT = 1e-3)
+ Threshold for convergence
+
+ optimizer: str, optional (DEFAULT = 'fp')
+ Method for optimization , either Expectation Maximization or
+ Fixed Point Gull-MacKay {'em','fp'}. Fixed point iterations are
+ faster, but can be numerically unstable (especially in case of near perfect fit).
+
+ fit_intercept: bool, optional (DEFAULT = True)
+ If True includes bias term in model
+
+ perfect_fit_tol: float (DEAFAULT = 1e-5)
+ Prevents overflow of precision parameters (this is smallest value RSS can have).
+ ( !!! Note if using EM instead of fixed-point, try smaller values
+ of perfect_fit_tol, for better estimates of variance of predictive distribution )
+
+ alpha: float (DEFAULT = 1)
+ Initial value of precision paramter for coefficients ( by default we define
+ very broad distribution )
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied, otherwise will be
+
+ verbose: bool, optional (Default = False)
+ If True at each iteration progress report is printed out
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ intercept_: float
+ Value of bias term (if fit_intercept is False, then intercept_ = 0)
+
+ alpha_ : float
+ Estimated precision of coefficients
+
+ beta_ : float
+ Estimated precision of noise
+
+ eigvals_ : array, shape = (n_features, )
+ Eigenvalues of covariance matrix (from posterior distribution of weights)
+
+ eigvecs_ : array, shape = (n_features, n_featues)
+ Eigenvectors of covariance matrix (from posterior distribution of weights)
+
+ '''
+
+ def __init__(self,n_iter = 300, tol = 1e-3, optimizer = 'fp', fit_intercept = True,
+ normalize=True, perfect_fit_tol = 1e-6, alpha = 1, copy_X = True, verbose = False):
+ super(EBLinearRegression,self).__init__(n_iter, tol, fit_intercept, copy_X, verbose)
+ if optimizer not in ['em','fp']:
+ raise ValueError('Optimizer can be either "em" or "fp" ')
+ self.optimizer = optimizer
+ self.alpha = alpha
+ self.perfect_fit = False
+ self.normalize = True
+ self.scores_ = [np.NINF]
+ self.perfect_fit_tol = perfect_fit_tol
+
+ def _check_convergence(self, mu, mu_old):
+ '''
+ Checks convergence of algorithm using changes in mean of posterior
+ distribution of weights
+ '''
+ return np.sum(abs(mu-mu_old)>self.tol) == 0
+
+
+ def _center_data(self,X,y):
+ ''' Centers data'''
+ X = as_float_array(X,copy = self.copy_X)
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype = X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X, axis=0)
+ X -= X_mean
+ if self.normalize:
+ X, X_std = f_normalize(X, axis=0, copy=False,
+ return_norm=True)
+ else:
+ X_std = np.ones(X.shape[1], dtype=X.dtype)
+ y_mean = np.average(y, axis=0)
+ y = y - y_mean
+ else:
+ X_mean = np.zeros(X.shape[1],dtype = X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X,y, X_mean, y_mean, X_std
+
+ def fit(self, X, y):
+ '''
+ Fits Bayesian Linear Regression using Empirical Bayes
+
+ Parameters
+ ----------
+ X: array-like of size [n_samples,n_features]
+ Matrix of explanatory variables (should not include bias term)
+
+ y: array-like of size [n_features]
+ Vector of dependent variables.
+
+ Returns
+ -------
+ object: self
+ self
+
+ '''
+ # preprocess data
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # precision of noise & and coefficients
+ alpha = self.alpha
+ var_y = np.var(y)
+ # check that variance is non zero !!!
+ if var_y == 0 :
+ beta = 1e-2
+ else:
+ beta = 1. / np.var(y)
+
+ # to speed all further computations save svd decomposition and reuse it later
+ u,d,vt = svd(X, full_matrices = False)
+ Uy = np.dot(u.T,y)
+ dsq = d**2
+ mu = 0
+
+ for i in range(self.n_iter):
+
+ # find mean for posterior of w ( for EM this is E-step)
+ mu_old = mu
+ if n_samples > n_features:
+ mu = vt.T * d/(dsq+alpha/beta)
+ else:
+ # clever use of SVD here , faster for large n_features
+ mu = u * 1./(dsq + alpha/beta)
+ mu = np.dot(X.T,mu)
+ mu = np.dot(mu,Uy)
+
+ # precompute errors, since both methods use it in estimation
+ error = y - np.dot(X,mu)
+ sqdErr = np.sum(error**2)
+
+ if sqdErr / n_samples < self.perfect_fit_tol:
+ self.perfect_fit = True
+ warnings.warn( ('Almost perfect fit!!! Estimated values of variance '
+ 'for predictive distribution are computed using only RSS'))
+ break
+
+ if self.optimizer == "fp":
+ gamma = np.sum(beta*dsq/(beta*dsq + alpha))
+ # use updated mu and gamma parameters to update alpha and beta
+ # !!! made computation numerically stable for perfect fit case
+ alpha = gamma / (np.sum(mu**2) + np.finfo(np.float32).eps )
+ beta = ( n_samples - gamma ) / (sqdErr + np.finfo(np.float32).eps )
+ else:
+ # M-step, update parameters alpha and beta to maximize ML TYPE II
+ eigvals = 1. / (beta * dsq + alpha)
+ alpha = n_features / ( np.sum(mu**2) + np.sum(1/eigvals) )
+ beta = n_samples / ( sqdErr + np.sum(dsq/eigvals) )
+
+ # if converged or exceeded maximum number of iterations => terminate
+ converged = self._check_convergence(mu_old,mu)
+ if self.verbose:
+ print( "Iteration {0} completed".format(i) )
+ if converged is True:
+ print("Algorithm converged after {0} iterations".format(i))
+ if converged or i==self.n_iter -1:
+ break
+ eigvals = 1./(beta * dsq + alpha)
+ self.coef_ = beta*np.dot(vt.T*d*eigvals ,Uy)
+ self._set_intercept(X_mean,y_mean,X_std)
+ self.beta_ = beta
+ self.alpha_ = alpha
+ self.eigvals_ = eigvals
+ self.eigvecs_ = vt.T
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self,X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of predictive
+ distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ '''
+ y_hat = np.dot(X,self.coef_) + self.intercept_
+
+ if return_std:
+ if self.normalize:
+ X = (X - self._x_mean_) / self._x_std
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ std_hat = np.sqrt(var_pred)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+
+# ============================== VBLR =========================================
+
+def gamma_mean(a,b):
+ '''
+ Computes mean of gamma distribution
+
+ Parameters
+ ----------
+ a: float
+ Shape parameter of Gamma distribution
+
+ b: float
+ Rate parameter of Gamma distribution
+
+ Returns
+ -------
+ : float
+ Mean of Gamma distribution
+ '''
+ return float(a) / b
+
+
+
+class VBLinearRegression(BayesianLinearRegression):
+ '''
+ Implements Bayesian Linear Regression using mean-field approximation.
+ Assumes gamma prior on precision parameters of coefficients and noise.
+
+ Parameters:
+ -----------
+ n_iter: int, optional (DEFAULT = 100)
+ Maximum number of iterations for KL minimization
+
+ tol: float, optional (DEFAULT = 1e-3)
+ Convergence threshold
+
+ fit_intercept: bool, optional (DEFAULT = True)
+ If True will use bias term in model fitting
+
+ a: float, optional (Default = 1e-4)
+ Shape parameter of Gamma prior for precision of coefficients
+
+ b: float, optional (Default = 1e-4)
+ Rate parameter of Gamma prior for precision coefficients
+
+ c: float, optional (Default = 1e-4)
+ Shape parameter of Gamma prior for precision of noise
+
+ d: float, optional (Default = 1e-4)
+ Rate parameter of Gamma prior for precision of noise
+
+ verbose: bool, optional (Default = False)
+ If True at each iteration progress report is printed out
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ intercept_: float
+ Value of bias term (if fit_intercept is False, then intercept_ = 0)
+
+ alpha_ : float
+ Mean of precision of coefficients
+
+ beta_ : float
+ Mean of precision of noise
+
+ eigvals_ : array, shape = (n_features, )
+ Eigenvalues of covariance matrix (from posterior distribution of weights)
+
+ eigvecs_ : array, shape = (n_features, n_featues)
+ Eigenvectors of covariance matrix (from posterior distribution of weights)
+
+ '''
+
+ def __init__(self, n_iter = 100, tol =1e-4, fit_intercept = True,
+ a = 1e-4, b = 1e-4, c = 1e-4, d = 1e-4, copy_X = True,
+ verbose = False):
+ super(VBLinearRegression,self).__init__(n_iter, tol, fit_intercept, copy_X,
+ verbose)
+ self.a,self.b = a, b
+ self.c,self.d = c, d
+
+
+ def fit(self,X,y):
+ '''
+ Fits Variational Bayesian Linear Regression Model
+
+ Parameters
+ ----------
+ X: array-like of size [n_samples,n_features]
+ Matrix of explanatory variables (should not include bias term)
+
+ Y: array-like of size [n_features]
+ Vector of dependent variables.
+
+ Returns
+ -------
+ object: self
+ self
+ '''
+ # preprocess data
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # SVD decomposition, done once , reused at each iteration
+ u,D,vt = svd(X, full_matrices = False)
+ dsq = D**2
+ UY = np.dot(u.T,y)
+
+ # some parameters of Gamma distribution have closed form solution
+ a = self.a + 0.5 * n_features
+ c = self.c + 0.5 * n_samples
+ b,d = self.b, self.d
+
+ # initial mean of posterior for coefficients
+ mu = 0
+
+ for i in range(self.n_iter):
+
+ # update parameters of distribution Q(weights)
+ e_beta = gamma_mean(c,d)
+ e_alpha = gamma_mean(a,b)
+ mu_old = np.copy(mu)
+ mu,eigvals = self._posterior_weights(e_beta,e_alpha,UY,dsq,u,vt,D,X)
+
+ # update parameters of distribution Q(precision of weights)
+ b = self.b + 0.5*( np.sum(mu**2) + np.sum(eigvals))
+
+ # update parameters of distribution Q(precision of likelihood)
+ sqderr = np.sum((y - np.dot(X,mu))**2)
+ xsx = np.sum(dsq*eigvals)
+ d = self.d + 0.5*(sqderr + xsx)
+
+ # check convergence
+ converged = self._check_convergence(mu,mu_old)
+ if self.verbose is True:
+ print("Iteration {0} is completed".format(i))
+ if converged is True:
+ print("Algorithm converged after {0} iterations".format(i))
+
+ # terminate if convergence or maximum number of iterations are achieved
+ if converged or i==(self.n_iter-1):
+ break
+
+ # save necessary parameters
+ self.beta_ = gamma_mean(c,d)
+ self.alpha_ = gamma_mean(a,b)
+ self.coef_, self.eigvals_ = self._posterior_weights(self.beta_, self.alpha_, UY,
+ dsq, u, vt, D, X)
+ self._set_intercept(X_mean,y_mean,X_std)
+ self.eigvecs_ = vt.T
+ return self
+
+
+ def _posterior_weights(self, e_beta, e_alpha, UY, dsq, u, vt, d, X):
+ '''
+ Calculates parameters of approximate posterior distribution
+ of weights
+ '''
+ # eigenvalues of covariance matrix
+ sigma = 1./ (e_beta*dsq + e_alpha)
+
+ # mean of approximate posterior distribution
+ n_samples, n_features = X.shape
+ if n_samples > n_features:
+ mu = vt.T * d/(dsq + e_alpha/e_beta)# + np.finfo(np.float64).eps)
+ else:
+ mu = u * 1./(dsq + e_alpha/e_beta)# + np.finfo(np.float64).eps)
+ mu = np.dot(X.T,mu)
+ mu = np.dot(mu,UY)
+ return mu,sigma
+
+ def predict(self,X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of predictive
+ distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ '''
+ x = (X - self._x_mean_) / self._x_std
+ y_hat = np.dot(x,self.coef_) + self._y_mean
+
+ if return_std:
+ data_noise = 1./self.beta_
+ model_noise = np.sum(np.dot(X,self.eigvecs_)**2 * self.eigvals_,1)
+ var_pred = data_noise + model_noise
+ std_hat = np.sqrt(var_pred)
+ return y_hat, std_hat
+ else:
+ return y_hat
\ No newline at end of file
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/engine.py b/examples/model-comparison/bayesvalidrox/surrogate_models/engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..42307d4770d4ae23a40107dfea64057aac682c23
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/engine.py
@@ -0,0 +1,2225 @@
+# -*- coding: utf-8 -*-
+"""
+Engine to train the surrogate
+
+"""
+import copy
+from copy import deepcopy, copy
+import h5py
+import joblib
+import numpy as np
+import os
+
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import pandas as pd
+import sys
+import seaborn as sns
+from joblib import Parallel, delayed
+
+
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+from bayesvalidrox.bayes_inference.discrepancy import Discrepancy
+from .exploration import Exploration
+import pathlib
+
+#from .inputs import Input
+#from .exp_designs import ExpDesigns
+#from .surrogate_models import MetaModel
+#from bayesvalidrox.post_processing.post_processing import PostProcessing
+
+def hellinger_distance(P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ P = np.array(P)
+ Q= np.array(Q)
+
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+
+def logpdf(x, mean, cov):
+ """
+ Computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+def subdomain(Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : int
+ Number of samples to divide the domain for.
+ n_params : int
+ The number of params to build the subdomains for
+
+ Returns
+ -------
+ Subdomains : List of tuples of tuples
+ Each tuple of tuples divides one set of bounds into n_new_samples parts.
+
+ """
+ n_params = len(Bounds)
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+class Engine():
+
+
+ def __init__(self, MetaMod, Model, ExpDes):
+ self.MetaModel = MetaMod
+ self.Model = Model
+ self.ExpDesign = ExpDes
+ self.parallel = False
+
+ def start_engine(self) -> None:
+ """
+ Do all the preparations that need to be run before the actual training
+
+ Returns
+ -------
+ None
+
+ """
+ self.out_names = self.Model.Output.names
+ self.MetaModel.out_names = self.out_names
+
+
+ def train_normal(self, parallel = False, verbose = False, save = False) -> None:
+ """
+ Trains surrogate on static samples only.
+ Samples are taken from the experimental design and the specified
+ model is run on them.
+ Alternatively the samples can be read in from a provided hdf5 file.
+
+
+ Returns
+ -------
+ None
+
+ """
+
+ ExpDesign = self.ExpDesign
+ MetaModel = self.MetaModel
+
+ # Read ExpDesign (training and targets) from the provided hdf5
+ if ExpDesign.hdf5_file is not None:
+ # TODO: need to run 'generate_ED' as well after this or not?
+ ExpDesign.read_from_file(self.out_names)
+ else:
+ # Check if an old hdf5 file exists: if yes, rename it
+ hdf5file = f'ExpDesign_{self.Model.name}.hdf5'
+ if os.path.exists(hdf5file):
+ # os.rename(hdf5file, 'old_'+hdf5file)
+ file = pathlib.Path(hdf5file)
+ file.unlink()
+
+ # Prepare X samples
+ # For training the surrogate use ExpDesign.X_tr, ExpDesign.X is for the model to run on
+ ExpDesign.generate_ED(ExpDesign.n_init_samples,
+ transform=True,
+ max_pce_deg=np.max(MetaModel.pce_deg))
+
+ # Run simulations at X
+ if not hasattr(ExpDesign, 'Y') or ExpDesign.Y is None:
+ print('\n Now the forward model needs to be run!\n')
+ ED_Y, up_ED_X = self.Model.run_model_parallel(ExpDesign.X, mp = parallel)
+ ExpDesign.Y = ED_Y
+ else:
+ # Check if a dict has been passed.
+ if not type(ExpDesign.Y) is dict:
+ raise Exception('Please provide either a dictionary or a hdf5'
+ 'file to ExpDesign.hdf5_file argument.')
+
+ # Separate output dict and x-values
+ if 'x_values' in ExpDesign.Y:
+ ExpDesign.x_values = ExpDesign.Y['x_values']
+ del ExpDesign.Y['x_values']
+ else:
+ print('No x_values are given, this might lead to issues during PostProcessing')
+
+
+ # Fit the surrogate
+ MetaModel.fit(ExpDesign.X, ExpDesign.Y, parallel, verbose)
+
+ # Save what there is to save
+ if save:
+ # Save surrogate
+ with open(f'surrogates/surrogate_{self.Model.name}.pk1', 'wb') as output:
+ joblib.dump(MetaModel, output, 2)
+
+ # Zip the model run directories
+ if self.Model.link_type.lower() == 'pylink' and\
+ self.ExpDesign.sampling_method.lower() != 'user':
+ self.Model.zip_subdirs(self.Model.name, f'{self.Model.name}_')
+
+
+ def train_sequential(self, parallel = False, verbose = False) -> None:
+ """
+ Train the surrogate in a sequential manner.
+ First build and train evereything on the static samples, then iterate
+ choosing more samples and refitting the surrogate on them.
+
+ Returns
+ -------
+ None
+
+ """
+ #self.train_normal(parallel, verbose)
+ self.parallel = parallel
+ self.train_seq_design(parallel, verbose)
+
+
+ # -------------------------------------------------------------------------
+ def eval_metamodel(self, samples=None, nsamples=None,
+ sampling_method='random', return_samples=False):
+ """
+ Evaluates meta-model at the requested samples. One can also generate
+ nsamples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate meta-model at. The default is None.
+ nsamples : int, optional
+ Number of samples to generate, if no `samples` is provided. The
+ default is None.
+ sampling_method : str, optional
+ Type of sampling, if no `samples` is provided. The default is
+ 'random'.
+ return_samples : bool, optional
+ Retun samples, if no `samples` is provided. The default is False.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean of the predictions.
+ std_pred : dict
+ Standard deviatioon of the predictions.
+ """
+ # Generate or transform (if need be) samples
+ if samples is None:
+ # Generate
+ samples = self.ExpDesign.generate_samples(
+ nsamples,
+ sampling_method
+ )
+
+ # Transformation to other space is to be done in the MetaModel
+ # TODO: sort the transformations better
+ mean_pred, std_pred = self.MetaModel.eval_metamodel(samples)
+
+ if return_samples:
+ return mean_pred, std_pred, samples
+ else:
+ return mean_pred, std_pred
+
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self, parallel = False, verbose = False):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ self.parallel = parallel
+
+ # Initialization
+ self.SeqModifiedLOO = {}
+ self.seqValidError = {}
+ self.SeqBME = {}
+ self.SeqKLD = {}
+ self.SeqDistHellinger = {}
+ self.seqRMSEMean = {}
+ self.seqRMSEStd = {}
+ self.seqMinDist = []
+
+ if not hasattr(self.MetaModel, 'valid_samples'):
+ self.ExpDesign.valid_samples = []
+ self.ExpDesign.valid_model_runs = []
+ self.valid_likelihoods = []
+
+ validError = None
+
+
+ # Determine the metamodel type
+ if self.MetaModel.meta_model_type.lower() != 'gpe':
+ pce = True
+ else:
+ pce = False
+ mc_ref = True if bool(self.Model.mc_reference) else False
+ if mc_ref:
+ self.Model.read_observation('mc_ref')
+
+ # Get the parameters
+ max_n_samples = self.ExpDesign.n_max_samples
+ mod_LOO_threshold = self.ExpDesign.mod_LOO_threshold
+ n_canddidate = self.ExpDesign.n_canddidate
+ post_snapshot = self.ExpDesign.post_snapshot
+ n_replication = self.ExpDesign.n_replication
+ util_func = self.ExpDesign.util_func
+ output_name = self.out_names
+
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [self.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ # TODO: recheck the logic in this if statement
+ if (len(self.Model.observations) != 0 or self.Model.meas_file is not None) and hasattr(self.MetaModel, 'Discrepancy'):
+ self.observations = self.Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ # TODO: TotalSigma2 not defined if not in this else???
+ # TODO: no self.observations if in here
+ TotalSigma2 = {}
+
+ # ---------- Initial self.MetaModel ----------
+ self.train_normal(parallel = parallel, verbose=verbose)
+
+ initMetaModel = deepcopy(self.MetaModel)
+
+ # Validation error if validation set is provided.
+ if self.ExpDesign.valid_model_runs:
+ init_rmse, init_valid_error = self._validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ TotalSigma2 = self.MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self._BME_Calculator(
+ obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = self.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self._error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean:.2f},"
+ f" {init_rmse_std:.2f}")
+
+ # Read the initial experimental design
+ Xinit = self.ExpDesign.X
+ init_n_samples = len(self.ExpDesign.X)
+ initYprev = self.ExpDesign.Y#initMetaModel.ModelOutputDict
+ #self.MetaModel.ModelOutputDict = self.ExpDesign.Y
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ ## Get some initial statistics
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ #prevExpDesign_dict = {}
+ # Can run sequential design multiple times for comparison
+ for repIdx in range(n_replication):
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # util_func: the function to use inside the type of exploitation
+ for util_f in util_func:
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ self.ExpDesign.X = Xinit
+ self.ExpDesign.Y = initYprev
+ self.ExpDesign.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(self.MetaModel)
+ #prevExpDesign_dict[itr_no] = deepcopy(self.ExpDesign)
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel(
+ samples=Xfull[-1].reshape(1, -1))
+ del prevMetaModel_dict[itr_no-1]
+
+ # Optimal Bayesian Design
+ #self.MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.choose_next_sample(TotalSigma2,
+ n_canddidate,
+ util_f)
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ self.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = self.Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(self.ExpDesign, 'adapt_verbose') and \
+ self.ExpDesign.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = self.MetaModel.eval_metamodel(
+ samples=Xnew
+ )
+ adaptPlot(
+ self.MetaModel, Ynew, y_hat, std_hat,
+ plotED=False
+ )
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ self.ExpDesign.Y[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ self.ExpDesign.sampling_method = 'user'
+ self.ExpDesign.X = Xfull
+ #self.ExpDesign.Y = self.MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = self.ExpDesign.Y
+
+ # Pass the new prior as the input
+ # TODO: another look at this - no difference apc to pce to gpe?
+ self.MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ self.MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ self.MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ self.MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ self.train_normal(parallel=False)
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if self.ExpDesign.valid_model_runs:
+ rmse, validError = self._validError(self.MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self._BME_Calculator(obs_data, TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = self.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = self.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self._error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ self.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(self.ExpDesign.valid_model_runs) != 0:
+ self.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ self.SeqBME[strKey] = SeqBME
+ self.SeqKLD[strKey] = SeqKLD
+ if hasattr(self.MetaModel, 'valid_likelihoods') and \
+ self.valid_likelihoods:
+ self.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ self.seqRMSEMean[strKey] = seqRMSEMean
+ self.seqRMSEStd[strKey] = seqRMSEStd
+
+ # return self.MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = self.ExpDesign.X
+ out_dict_y = self.ExpDesign.Y
+ out_names = self.out_names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, y_hat, std, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Get the data
+ obs_data = self.observations
+ # TODO: this should be optimizable to be calculated explicitly
+ if hasattr(self.Model, 'n_obs'):
+ n_obs = self.Model.n_obs
+ else:
+ n_obs = self.n_obs
+ mc_size = 10000
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((mc_size))
+ # print(y_hat)
+ # print(list[y_hat])
+ for key in list(y_hat):
+ cov = np.diag(std[key]**2)
+ # print(y_hat[key], cov)
+ # TODO: added the allow_singular = True here
+ rv = stats.multivariate_normal(mean=y_hat[key], cov=cov,)
+ Y_MC[key] = rv.rvs(size=mc_size)
+ logPriorLikelihoods += rv.logpdf(Y_MC[key])
+ std_MC[key] = np.zeros((mc_size, y_hat[key].shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self._normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, mc_size)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods), dtype=np.longdouble)#float128)
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = np.nanmean(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self._normpdf(
+ y_hat, std, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ MetaModel = self.MetaModel
+ out_names = self.out_names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = self.ExpDesign.X
+ oldExpDesignY = self.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+ PCE_Model_can = deepcopy(MetaModel)
+ engine_can = deepcopy(self)
+ # Add the candidate to the ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ NewExpDesignY[key] = np.vstack(
+ (oldExpDesignY[key], Y_PC_can[key])
+ )
+
+ engine_can.ExpDesign.sampling_method = 'user'
+ engine_can.ExpDesign.X = NewExpDesignX
+ #engine_can.ModelOutputDict = NewExpDesignY
+ engine_can.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ engine_can.MetaModel.input_obj.poly_coeffs_flag = False
+ engine_can.start_engine()
+ engine_can.train_normal(parallel=False)
+ engine_can.MetaModel.fit(NewExpDesignX, NewExpDesignY)
+# engine_can.train_norm_design(parallel=False)
+
+ # Set the ExpDesign to its original values
+ engine_can.ExpDesign.X = oldExpDesignX
+ engine_can.ModelOutputDict = oldExpDesignY
+ engine_can.ExpDesign.Y = oldExpDesignY
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = engine_can.MetaModel.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = engine_can.MetaModel.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self._normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.longdouble))#float128))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postLikelihoods /= np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ # Evaluate all candidates
+ y_can, std_can = self.MetaModel.eval_metamodel(samples=candidates)
+ # loop through candidates
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="BAL Design"):
+ y_hat = {key: items[idx] for key, items in y_can.items()}
+ std = {key: items[idx] for key, items in std_can.items()}
+
+ # print(y_hat)
+ # print(std)
+ U_J_d[idx] = self.util_BayesianActiveDesign(
+ y_hat, std, sigma2Dict, var)
+
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithm to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"Global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradeoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.ExpDesign.n_init_samples
+ n_max_samples = self.ExpDesign.n_max_samples
+
+ itrNumber = (self.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.ExpDesign.n_init_samples
+ n_max_samples = self.ExpDesign.n_max_samples
+ itrNumber = (self.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def choose_next_sample(self, sigma2=None, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None. # TODO: default is set to DKL, not none
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ Bounds = self.ExpDesign.bound_tuples
+ n_new_samples = self.ExpDesign.n_new_samples
+ explore_method = self.ExpDesign.explore_method
+ exploit_method = self.ExpDesign.exploit_method
+ n_cand_groups = self.ExpDesign.n_cand_groups
+ tradeoff_scheme = self.ExpDesign.tradeoff_scheme
+
+ old_EDX = self.ExpDesign.X
+ old_EDY = self.ExpDesign.Y.copy()
+ ndim = self.ExpDesign.X.shape[1]
+ OutputNames = self.out_names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+ if not hasattr(self.ExpDesign, 'ExploitFunction'):
+ raise AttributeError('Function `ExploitFunction` not given to the ExpDesign, thus cannor run user-defined sequential scheme')
+ # TODO: syntax does not fully match the rest - can test this??
+ Xnew, filteredSamples = self.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+
+ # Dual-Annealing works differently from the rest, so deal with this first
+ # Here exploration and exploitation are performed simulataneously
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ subdomains = subdomain(Bounds, n_new_samples)
+
+ # Multiprocessing
+ if self.parallel:
+ args = []
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+ # Without multiprocessing
+ else:
+ results = []
+ for i in range(n_new_samples):
+ results.append(self.dual_annealing(exploit_method, subdomains[i], sigma2, var, i))
+
+ # New sample
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+ print("\nXnew:\n", Xnew)
+
+ # Computational cost
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"Elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ return Xnew, None
+
+ # Generate needed Exploration class
+ explore = Exploration(self.ExpDesign, n_candidates)
+ explore.w = 100 # * ndim #500 # TODO: where does this value come from?
+
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+
+ # Generate the candidate samples
+ # TODO: here use the sampling method provided by the expdesign?
+ sampling_method = self.ExpDesign.sampling_method
+
+ # TODO: changed this from 'random' for LOOCV
+ if explore_method == 'LOOCV':
+ allCandidates = self.ExpDesign.generate_samples(n_candidates,
+ sampling_method)
+ else:
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = self.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = self.MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(self.ExpDesign, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradeoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+ # Check if all needed properties are set
+ if not hasattr(self.ExpDesign, 'max_func_itr'):
+ raise AttributeError('max_func_itr not given to the experimental design')
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = self.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = self.ExpDesign.generate_samples(
+ n_candidates, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+ # print(candidates)
+ # print(split_cand)
+ if self.parallel:
+ results = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ else:
+ results = []
+ for i in range(n_cand_groups):
+ results.append(self.run_util_func(exploit_method, split_cand[i], i, sigma2, var, X_MC))
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Accumulate the samples
+ finalCandidates = np.concatenate((allCandidates, candidates), axis = 0)
+ finalCandidates = np.unique(finalCandidates, axis = 0)
+
+ # Calculations take into account both exploration and exploitation
+ # samples without duplicates
+ totalScore = np.zeros(finalCandidates.shape[0])
+ #self.totalScore = totalScore
+
+ for cand_idx in range(finalCandidates.shape[0]):
+ # find candidate indices
+ idx1 = np.where(allCandidates == finalCandidates[cand_idx])[0]
+ idx2 = np.where(candidates == finalCandidates[cand_idx])[0]
+
+ # exploration
+ if idx1 != []:
+ idx1 = idx1[0]
+ totalScore[cand_idx] += explore_w * scoreExploration[idx1]
+
+ # exploitation
+ if idx2 != []:
+ idx2 = idx2[0]
+ totalScore[cand_idx] += exploit_w * norm_U_J_d[idx2]
+
+
+ # Total score
+ totalScore = exploit_w * norm_U_J_d
+ totalScore += explore_w * scoreExploration
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ # Changed this from allCandiates to full set of candidates
+ # TODO: still not changed for e.g. 'Voronoi'
+ Xnew = finalCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradeoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # print(UtilMethod)
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = self.MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+ # print(allModifiedLOO.shape)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ # print(totalScore.shape)
+ # print(explore_w)
+ # print(scoreExploration.shape)
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+
+ # TODO: why does it also return None?
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self # TODO: this doesn't fully seem correct?
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = self.out_names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = self.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ # TODO: this is just a patch, need to look at again!
+ Scores = list(self.MetaModel.score_dict['b_1'][OutputName].values())
+ #print(Scores)
+ #print(self.MetaModel.score_dict)
+ #print(self.MetaModel.score_dict.values())
+ #print(self.MetaModel.score_dict['b_1'].values())
+ #print(self.MetaModel.score_dict['b_1'][OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ # TODO: also patched here
+ BasisIndices = self.MetaModel.basis_dict['b_1'][OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = self.MetaModel.univ_basis_vals(oldExpDesignX)
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.MetaModel.univ_basis_vals(candidates)
+ Psi_c = self.MetaModel.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var.lower() == 'd-opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var.lower() == 'a-opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var.lower() == 'k-opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ # print(var.lower())
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def _normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+ """
+ Calculated gaussian likelihood for given y+std based on given obs+sigma
+ # TODO: is this understanding correct?
+
+ Parameters
+ ----------
+ y_hat_pce : dict of 2d np arrays
+ Mean output of the surrogate.
+ std_pce : dict of 2d np arrays
+ Standard deviation output of the surrogate.
+ obs_data : dict of 1d np arrays
+ Observed data.
+ total_sigma2s : pandas dataframe, matches obs_data
+ Estimated uncertainty for the observed data.
+ rmse : dict, optional
+ RMSE values from validation of the surrogate. The default is None.
+
+ Returns
+ -------
+ likelihoods : dict of float
+ The likelihood for each surrogate eval in y_hat_pce compared to the
+ observations (?).
+
+ """
+
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(self.out_names):
+
+ # (Meta)Model Output
+ # print(y_hat_pce[out])
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+ else:
+ tot_sigma2s += np.mean(std_pce[out])**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+
+ # TODO: remove this here
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def _corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = self.ExpDesign.X # valid_samples
+ model_outputs = self.ExpDesign.Y # valid_model_runs
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = self.out_names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def _posteriorPlot(self, posterior, par_names, key):
+ """
+ Plot the posterior of a specific key as a corner plot
+
+ Parameters
+ ----------
+ posterior : 2d np.array
+ Samples of the posterior.
+ par_names : list of strings
+ List of the parameter names.
+ key : string
+ Output key that this posterior belongs to.
+
+ Returns
+ -------
+ figPosterior : corner.corner
+ Plot of the posterior.
+
+ """
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.ExpDesign.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+
+ # -------------------------------------------------------------------------
+ def _BME_Calculator(self, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ Parameters
+ ----------
+ obs_data : dict of 1d np arrays
+ Observed data.
+ sigma2Dict : pandas dataframe, matches obs_data
+ Estimated uncertainty for the observed data.
+ rmse : dict of floats, optional
+ RMSE values for each output-key. The dafault is None.
+
+ Returns
+ -------
+ (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ """
+ # Initializations
+ if hasattr(self, 'valid_likelihoods'):
+ valid_likelihoods = self.valid_likelihoods
+ else:
+ valid_likelihoods = []
+ valid_likelihoods = np.array(valid_likelihoods)
+
+ post_snapshot = self.ExpDesign.post_snapshot
+ if post_snapshot or valid_likelihoods.shape[0] != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = self.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ Y_MC, std_MC = self.MetaModel.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self._normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(self.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or valid_likelihoods:
+ # Hellinger distance
+ valid_likelihoods = np.array(valid_likelihoods)
+ ref_like = np.log(valid_likelihoods[(valid_likelihoods > 0)])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and self.MetaModel.n_params == 2 and not idx % 5:
+ BayesOpts = BayesInference(self)
+
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ # TODO: check with Farid if this first line is how it should be
+ BayesOpts.measured_data = obs_data
+ obs_data = pd.DataFrame(obs_data, columns=self.out_names)
+ BayesOpts.measurement_error = obs_data
+ # TODO: shouldn't the uncertainty be sigma2Dict instead of obs_data?
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def _validError(self):
+ """
+ Evaluate the metamodel on the validation samples and calculate the
+ error against the corresponding model runs
+
+ Returns
+ -------
+ rms_error : dict
+ RMSE for each validation run.
+ valid_error : dict
+ Normed (?)RMSE for each validation run.
+
+ """
+ # Extract the original model with the generated samples
+ valid_model_runs = self.ExpDesign.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ valid_PCE_runs, _ = self.MetaModel.eval_metamodel(samples=self.ExpDesign.valid_samples)
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in self.out_names:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def _error_Mean_Std(self):
+ """
+ Calculates the error in the overall mean and std approximation of the
+ surrogate against the mc-reference provided to the model.
+ This can only be applied to metamodels of polynomial type
+
+ Returns
+ -------
+ RMSE_Mean : float
+ RMSE of the means
+ RMSE_std : float
+ RMSE of the standard deviations
+
+ """
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self.MetaModel._compute_pce_moments()
+
+ # Compute the root mean squared error
+ for output in self.out_names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ self.Model.mc_reference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ self.Model.mc_reference['std'], pce_stds[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/eval_rec_rule.py b/examples/model-comparison/bayesvalidrox/surrogate_models/eval_rec_rule.py
new file mode 100644
index 0000000000000000000000000000000000000000..b583c7eb2ec58d55d19b34130812730d21a12368
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/eval_rec_rule.py
@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+
+
+Based on the implementation in UQLab [1].
+
+References:
+1. S. Marelli, and B. Sudret, UQLab: A framework for uncertainty quantification
+in Matlab, Proc. 2nd Int. Conf. on Vulnerability, Risk Analysis and Management
+(ICVRAM2014), Liverpool, United Kingdom, 2014, 2554-2563.
+
+2. S. Marelli, N. Lüthen, B. Sudret, UQLab user manual – Polynomial chaos
+expansions, Report # UQLab-V1.4-104, Chair of Risk, Safety and Uncertainty
+Quantification, ETH Zurich, Switzerland, 2021.
+
+Author: Farid Mohammadi, M.Sc.
+E-Mail: farid.mohammadi@iws.uni-stuttgart.de
+Department of Hydromechanics and Modelling of Hydrosystems (LH2)
+Institute for Modelling Hydraulic and Environmental Systems (IWS), University
+of Stuttgart, www.iws.uni-stuttgart.de/lh2/
+Pfaffenwaldring 61
+70569 Stuttgart
+
+Created on Fri Jan 14 2022
+"""
+import numpy as np
+from numpy.polynomial.polynomial import polyval
+
+
+def poly_rec_coeffs(n_max, poly_type, params=None):
+ """
+ Computes the recurrence coefficients for classical Wiener-Askey orthogonal
+ polynomials.
+
+ Parameters
+ ----------
+ n_max : int
+ Maximum polynomial degree.
+ poly_type : string
+ Polynomial type.
+ params : list, optional
+ Parameters required for `laguerre` poly type. The default is None.
+
+ Returns
+ -------
+ AB : dict
+ The 3 term recursive coefficients and the applicable ranges.
+
+ """
+
+ if poly_type == 'legendre':
+
+ def an(n):
+ return np.zeros((n+1, 1))
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = np.sqrt(1./(4-i**-2))
+ return sq_bn
+
+ bounds = [-1, 1]
+
+ elif poly_type == 'hermite':
+
+ def an(n):
+ return np.zeros((n+1, 1))
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = np.sqrt(i)
+ return sq_bn
+
+ bounds = [-np.inf, np.inf]
+
+ elif poly_type == 'laguerre':
+
+ def an(n):
+ a = np.zeros((n+1, 1))
+ for i in range(1, n+1):
+ a[i] = 2*n + params[1]
+ return a
+
+ def sqrt_bn(n):
+ sq_bn = np.zeros((n+1, 1))
+ sq_bn[0, 0] = 1
+ for i in range(1, n+1):
+ sq_bn[i, 0] = -np.sqrt(i * (i+params[1]-1))
+ return sq_bn
+
+ bounds = [0, np.inf]
+
+ AB = {'alpha_beta': np.concatenate((an(n_max), sqrt_bn(n_max)), axis=1),
+ 'bounds': bounds}
+
+ return AB
+
+
+def eval_rec_rule(x, max_deg, poly_type):
+ """
+ Evaluates the polynomial that corresponds to the Jacobi matrix defined
+ from the AB.
+
+ Parameters
+ ----------
+ x : array (n_samples)
+ Points where the polynomials are evaluated.
+ max_deg : int
+ Maximum degree.
+ poly_type : string
+ Polynomial type.
+
+ Returns
+ -------
+ values : array of shape (n_samples, max_deg+1)
+ Polynomials corresponding to the Jacobi matrix.
+
+ """
+ AB = poly_rec_coeffs(max_deg, poly_type)
+ AB = AB['alpha_beta']
+
+ values = np.zeros((len(x), AB.shape[0]+1))
+ values[:, 1] = 1 / AB[0, 1]
+
+ for k in range(AB.shape[0]-1):
+ values[:, k+2] = np.multiply((x - AB[k, 0]), values[:, k+1]) - \
+ np.multiply(values[:, k], AB[k, 1])
+ values[:, k+2] = np.divide(values[:, k+2], AB[k+1, 1])
+ return values[:, 1:]
+
+
+def eval_rec_rule_arbitrary(x, max_deg, poly_coeffs):
+ """
+ Evaluates the polynomial at sample array x.
+
+ Parameters
+ ----------
+ x : array (n_samples)
+ Points where the polynomials are evaluated.
+ max_deg : int
+ Maximum degree.
+ poly_coeffs : dict
+ Polynomial coefficients computed based on moments.
+
+ Returns
+ -------
+ values : array of shape (n_samples, max_deg+1)
+ Univariate Polynomials evaluated at samples.
+
+ """
+ values = np.zeros((len(x), max_deg+1))
+
+ for deg in range(max_deg+1):
+ values[:, deg] = polyval(x, poly_coeffs[deg]).T
+
+ return values
+
+
+def eval_univ_basis(x, max_deg, poly_types, apoly_coeffs=None):
+ """
+ Evaluates univariate regressors along input directions.
+
+ Parameters
+ ----------
+ x : array of shape (n_samples, n_params)
+ Training samples.
+ max_deg : int
+ Maximum polynomial degree.
+ poly_types : list of strings
+ List of polynomial types for all parameters.
+ apoly_coeffs : dict , optional
+ Polynomial coefficients computed based on moments. The default is None.
+
+ Returns
+ -------
+ univ_vals : array of shape (n_samples, n_params, max_deg+1)
+ Univariate polynomials for all degrees and parameters evaluated at x.
+
+ """
+ # Initilize the output array
+ n_samples, n_params = x.shape
+ univ_vals = np.zeros((n_samples, n_params, max_deg+1))
+
+ for i in range(n_params):
+
+ if poly_types[i] == 'arbitrary':
+ polycoeffs = apoly_coeffs[f'p_{i+1}']
+ univ_vals[:, i] = eval_rec_rule_arbitrary(x[:, i], max_deg,
+ polycoeffs)
+ else:
+ univ_vals[:, i] = eval_rec_rule(x[:, i], max_deg, poly_types[i])
+
+ return univ_vals
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/exp_designs.py b/examples/model-comparison/bayesvalidrox/surrogate_models/exp_designs.py
new file mode 100644
index 0000000000000000000000000000000000000000..fa03fe17d96fb2c1f19546b7b72fb2fd6dd1c13a
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/exp_designs.py
@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Experimental design with associated sampling methods
+"""
+
+import numpy as np
+import math
+import itertools
+import chaospy
+import scipy.stats as st
+from tqdm import tqdm
+import h5py
+import os
+
+from .apoly_construction import apoly_construction
+from .input_space import InputSpace
+
+# -------------------------------------------------------------------------
+def check_ranges(theta, ranges):
+ """
+ This function checks if theta lies in the given ranges.
+
+ Parameters
+ ----------
+ theta : array
+ Proposed parameter set.
+ ranges : nested list
+ List of the praremeter ranges.
+
+ Returns
+ -------
+ c : bool
+ If it lies in the given range, it return True else False.
+
+ """
+ c = True
+ # traverse in the list1
+ for i, bounds in enumerate(ranges):
+ x = theta[i]
+ # condition check
+ if x < bounds[0] or x > bounds[1]:
+ c = False
+ return c
+ return c
+
+
+class ExpDesigns(InputSpace):
+ """
+ This class generates samples from the prescribed marginals for the model
+ parameters using the `Input` object.
+
+ Attributes
+ ----------
+ Input : obj
+ Input object containing the parameter marginals, i.e. name,
+ distribution type and distribution parameters or available raw data.
+ meta_Model_type : str
+ Type of the meta_Model_type.
+ sampling_method : str
+ Name of the sampling method for the experimental design. The following
+ sampling method are supported:
+
+ * random
+ * latin_hypercube
+ * sobol
+ * halton
+ * hammersley
+ * chebyshev(FT)
+ * grid(FT)
+ * user
+ hdf5_file : str
+ Name of the hdf5 file that contains the experimental design.
+ n_new_samples : int
+ Number of (initial) training points.
+ n_max_samples : int
+ Number of maximum training points.
+ mod_LOO_threshold : float
+ The modified leave-one-out cross validation threshold where the
+ sequential design stops.
+ tradeoff_scheme : str
+ Trade-off scheme to assign weights to the exploration and exploitation
+ scores in the sequential design.
+ n_canddidate : int
+ Number of candidate training sets to calculate the scores for.
+ explore_method : str
+ Type of the exploration method for the sequential design. The following
+ methods are supported:
+
+ * Voronoi
+ * random
+ * latin_hypercube
+ * LOOCV
+ * dual annealing
+ exploit_method : str
+ Type of the exploitation method for the sequential design. The
+ following methods are supported:
+
+ * BayesOptDesign
+ * BayesActDesign
+ * VarOptDesign
+ * alphabetic
+ * Space-filling
+ util_func : str or list
+ The utility function to be specified for the `exploit_method`. For the
+ available utility functions see Note section.
+ n_cand_groups : int
+ Number of candidate groups. Each group of candidate training sets will
+ be evaulated separately in parallel.
+ n_replication : int
+ Number of replications. Only for comparison. The default is 1.
+ post_snapshot : int
+ Whether to plot the posterior in the sequential design. The default is
+ `True`.
+ step_snapshot : int
+ The number of steps to plot the posterior in the sequential design. The
+ default is 1.
+ max_a_post : list or array
+ Maximum a posteriori of the posterior distribution, if known. The
+ default is `[]`.
+ adapt_verbose : bool
+ Whether to plot the model response vs that of metamodel for the new
+ trining point in the sequential design.
+
+ Note
+ ----------
+ The following utiliy functions for the **exploitation** methods are
+ supported:
+
+ #### BayesOptDesign (when data is available)
+ - DKL (Kullback-Leibler Divergence)
+ - DPP (D-Posterior-percision)
+ - APP (A-Posterior-percision)
+
+ #### VarBasedOptDesign -> when data is not available
+ - Entropy (Entropy/MMSE/active learning)
+ - EIGF (Expected Improvement for Global fit)
+ - LOOCV (Leave-one-out Cross Validation)
+
+ #### alphabetic
+ - D-Opt (D-Optimality)
+ - A-Opt (A-Optimality)
+ - K-Opt (K-Optimality)
+ """
+
+ def __init__(self, Input, meta_Model_type='pce',
+ sampling_method='random', hdf5_file=None,
+ n_new_samples=1, n_max_samples=None, mod_LOO_threshold=1e-16,
+ tradeoff_scheme=None, n_canddidate=1, explore_method='random',
+ exploit_method='Space-filling', util_func='Space-filling',
+ n_cand_groups=4, n_replication=1, post_snapshot=False,
+ step_snapshot=1, max_a_post=[], adapt_verbose=False, max_func_itr=1):
+
+ self.InputObj = Input
+ self.meta_Model_type = meta_Model_type
+ self.sampling_method = sampling_method
+ self.hdf5_file = hdf5_file
+ self.n_new_samples = n_new_samples
+ self.n_max_samples = n_max_samples
+ self.mod_LOO_threshold = mod_LOO_threshold
+ self.explore_method = explore_method
+ self.exploit_method = exploit_method
+ self.util_func = util_func
+ self.tradeoff_scheme = tradeoff_scheme
+ self.n_canddidate = n_canddidate
+ self.n_cand_groups = n_cand_groups
+ self.n_replication = n_replication
+ self.post_snapshot = post_snapshot
+ self.step_snapshot = step_snapshot
+ self.max_a_post = max_a_post
+ self.adapt_verbose = adapt_verbose
+ self.max_func_itr = max_func_itr
+
+ # Other
+ self.apce = None
+ self.ndim = None
+
+ # Init
+ self.check_valid_inputs()
+
+ # -------------------------------------------------------------------------
+ def generate_samples(self, n_samples, sampling_method='random',
+ transform=False):
+ """
+ Generates samples with given sampling method
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+ sampling_method : str, optional
+ Sampling method. The default is `'random'`.
+ transform : bool, optional
+ Transformation via an isoprobabilistic transformation method. The
+ default is `False`.
+
+ Returns
+ -------
+ samples: array of shape (n_samples, n_params)
+ Generated samples from defined model input object.
+
+ """
+ try:
+ samples = chaospy.generate_samples(
+ int(n_samples), domain=self.origJDist, rule=sampling_method
+ )
+ except:
+ samples = self.random_sampler(int(n_samples)).T
+
+ return samples.T
+
+
+
+ # -------------------------------------------------------------------------
+ def generate_ED(self, n_samples, transform=False,
+ max_pce_deg=None):
+ """
+ Generates experimental designs (training set) with the given method.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested training points.
+ sampling_method : str, optional
+ Sampling method. The default is `'random'`.
+ transform : bool, optional
+ Isoprobabilistic transformation. The default is `False`.
+ max_pce_deg : int, optional
+ Maximum PCE polynomial degree. The default is `None`.
+
+ Returns
+ -------
+ None
+
+ """
+ if n_samples <0:
+ raise ValueError('A negative number of samples cannot be created. Please provide positive n_samples')
+ n_samples = int(n_samples)
+
+ if not hasattr(self, 'n_init_samples'):
+ self.n_init_samples = n_samples
+
+ # Generate the samples based on requested method
+ self.init_param_space(max_pce_deg)
+
+ sampling_method = self.sampling_method
+ # Pass user-defined samples as ED
+ if sampling_method == 'user':
+ if not hasattr(self, 'X'):
+ raise AttributeError('User-defined sampling cannot proceed as no samples provided. Please add them to this class as attribute X')
+ if not self.X.ndim == 2:
+ raise AttributeError('The provided samples shuld have 2 dimensions')
+ samples = self.X
+ self.n_samples = len(samples)
+
+ # Sample the distribution of parameters
+ elif self.input_data_given:
+ # Case II: Input values are directly given by the user.
+
+ if sampling_method == 'random':
+ samples = self.random_sampler(n_samples)
+
+ elif sampling_method == 'PCM' or \
+ sampling_method == 'LSCM':
+ samples = self.pcm_sampler(n_samples, max_pce_deg)
+
+ else:
+ # Create ExpDesign in the actual space using chaospy
+ try:
+ samples = chaospy.generate_samples(n_samples,
+ domain=self.JDist,
+ rule=sampling_method).T
+ except:
+ samples = self.JDist.resample(n_samples).T
+
+ elif not self.input_data_given:
+ # Case I = User passed known distributions
+ samples = chaospy.generate_samples(n_samples, domain=self.JDist,
+ rule=sampling_method).T
+
+ self.X = samples
+
+ def read_from_file(self, out_names):
+ """
+ Reads in the ExpDesign from a provided h5py file and saves the results.
+
+ Parameters
+ ----------
+ out_names : list of strings
+ The keys that are in the outputs (y) saved in the provided file.
+
+ Returns
+ -------
+ None.
+
+ """
+ if self.hdf5_file == None:
+ raise AttributeError('ExpDesign cannot be read in, please provide hdf5 file first')
+
+ # Read hdf5 file
+ f = h5py.File(self.hdf5_file, 'r+')
+
+ # Read EDX and pass it to ExpDesign object
+ try:
+ self.X = np.array(f["EDX/New_init_"])
+ except KeyError:
+ self.X = np.array(f["EDX/init_"])
+
+ # Update number of initial samples
+ self.n_init_samples = self.X.shape[0]
+
+ # Read EDX and pass it to ExpDesign object
+ self.Y = {}
+
+ # Extract x values
+ try:
+ self.Y["x_values"] = dict()
+ for varIdx, var in enumerate(out_names):
+ x = np.array(f[f"x_values/{var}"])
+ self.Y["x_values"][var] = x
+ except KeyError:
+ self.Y["x_values"] = np.array(f["x_values"])
+
+ # Store the output
+ for varIdx, var in enumerate(out_names):
+ try:
+ y = np.array(f[f"EDY/{var}/New_init_"])
+ except KeyError:
+ y = np.array(f[f"EDY/{var}/init_"])
+ self.Y[var] = y
+ f.close()
+ print(f'Experimental Design is read in from file {self.hdf5_file}')
+ print('')
+
+
+
+ # -------------------------------------------------------------------------
+ def random_sampler(self, n_samples, max_deg = None):
+ """
+ Samples the given raw data randomly.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+
+ max_deg : int, optional
+ Maximum degree. The default is `None`.
+ This will be used to run init_param_space, if it has not been done
+ until now.
+
+ Returns
+ -------
+ samples: array of shape (n_samples, n_params)
+ The sampling locations in the input space.
+
+ """
+ if not hasattr(self, 'raw_data'):
+ self.init_param_space(max_deg)
+ else:
+ if np.array(self.raw_data).ndim !=2:
+ raise AttributeError('The given raw data for sampling should have two dimensions')
+ samples = np.zeros((n_samples, self.ndim))
+ sample_size = self.raw_data.shape[1]
+
+ # Use a combination of raw data
+ if n_samples < sample_size:
+ for pa_idx in range(self.ndim):
+ # draw random indices
+ rand_idx = np.random.randint(0, sample_size, n_samples)
+ # store the raw data with given random indices
+ samples[:, pa_idx] = self.raw_data[pa_idx, rand_idx]
+ else:
+ try:
+ samples = self.JDist.resample(int(n_samples)).T
+ except AttributeError:
+ samples = self.JDist.sample(int(n_samples)).T
+ # Check if all samples are in the bound_tuples
+ for idx, param_set in enumerate(samples):
+ if not check_ranges(param_set, self.bound_tuples):
+ try:
+ proposed_sample = chaospy.generate_samples(
+ 1, domain=self.JDist, rule='random').T[0]
+ except:
+ proposed_sample = self.JDist.resample(1).T[0]
+ while not check_ranges(proposed_sample,
+ self.bound_tuples):
+ try:
+ proposed_sample = chaospy.generate_samples(
+ 1, domain=self.JDist, rule='random').T[0]
+ except:
+ proposed_sample = self.JDist.resample(1).T[0]
+ samples[idx] = proposed_sample
+
+ return samples
+
+ # -------------------------------------------------------------------------
+ def pcm_sampler(self, n_samples, max_deg):
+ """
+ Generates collocation points based on the root of the polynomial
+ degrees.
+
+ Parameters
+ ----------
+ n_samples : int
+ Number of requested samples.
+ max_deg : int
+ Maximum degree defined by user. Will also be used to run
+ init_param_space if that has not been done beforehand.
+
+ Returns
+ -------
+ opt_col_points: array of shape (n_samples, n_params)
+ Collocation points.
+
+ """
+
+ if not hasattr(self, 'raw_data'):
+ self.init_param_space(max_deg)
+
+ raw_data = self.raw_data
+
+ # Guess the closest degree to self.n_samples
+ def M_uptoMax(deg):
+ result = []
+ for d in range(1, deg+1):
+ result.append(math.factorial(self.ndim+d) //
+ (math.factorial(self.ndim) * math.factorial(d)))
+ return np.array(result)
+ #print(M_uptoMax(max_deg))
+ #print(np.where(M_uptoMax(max_deg) > n_samples)[0])
+
+ guess_Deg = np.where(M_uptoMax(max_deg) > n_samples)[0][0]
+
+ c_points = np.zeros((guess_Deg+1, self.ndim))
+
+ def PolynomialPa(parIdx):
+ return apoly_construction(self.raw_data[parIdx], max_deg)
+
+ for i in range(self.ndim):
+ poly_coeffs = PolynomialPa(i)[guess_Deg+1][::-1]
+ c_points[:, i] = np.trim_zeros(np.roots(poly_coeffs))
+
+ # Construction of optimal integration points
+ Prod = itertools.product(np.arange(1, guess_Deg+2), repeat=self.ndim)
+ sort_dig_unique_combos = np.array(list(filter(lambda x: x, Prod)))
+
+ # Ranking relatively mean
+ Temp = np.empty(shape=[0, guess_Deg+1])
+ for j in range(self.ndim):
+ s = abs(c_points[:, j]-np.mean(raw_data[j]))
+ Temp = np.append(Temp, [s], axis=0)
+ temp = Temp.T
+
+ index_CP = np.sort(temp, axis=0)
+ sort_cpoints = np.empty((0, guess_Deg+1))
+
+ for j in range(self.ndim):
+ #print(index_CP[:, j])
+ sort_cp = c_points[index_CP[:, j], j]
+ sort_cpoints = np.vstack((sort_cpoints, sort_cp))
+
+ # Mapping of Combination to Cpoint Combination
+ sort_unique_combos = np.empty(shape=[0, self.ndim])
+ for i in range(len(sort_dig_unique_combos)):
+ sort_un_comb = []
+ for j in range(self.ndim):
+ SortUC = sort_cpoints[j, sort_dig_unique_combos[i, j]-1]
+ sort_un_comb.append(SortUC)
+ sort_uni_comb = np.asarray(sort_un_comb)
+ sort_unique_combos = np.vstack((sort_unique_combos, sort_uni_comb))
+
+ # Output the collocation points
+ if self.sampling_method.lower() == 'lscm':
+ opt_col_points = sort_unique_combos
+ else:
+ opt_col_points = sort_unique_combos[0:self.n_samples]
+
+ return opt_col_points
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/exploration.py b/examples/model-comparison/bayesvalidrox/surrogate_models/exploration.py
new file mode 100644
index 0000000000000000000000000000000000000000..6abb652f145fadb410ecf8f987142e8ceb544a41
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/exploration.py
@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Exploration for sequential training of metamodels
+"""
+
+import numpy as np
+from scipy.spatial import distance
+
+
+class Exploration:
+ """
+ Created based on the Surrogate Modeling Toolbox (SUMO) [1].
+
+ [1] Gorissen, D., Couckuyt, I., Demeester, P., Dhaene, T. and Crombecq, K.,
+ 2010. A surrogate modeling and adaptive sampling toolbox for computer
+ based design. Journal of machine learning research.-Cambridge, Mass.,
+ 11, pp.2051-2055. sumo@sumo.intec.ugent.be - http://sumo.intec.ugent.be
+
+ Attributes
+ ----------
+ ExpDesign : obj
+ ExpDesign object.
+ n_candidate : int
+ Number of candidate samples.
+ mc_criterion : str
+ Selection crieterion. The default is `'mc-intersite-proj-th'`. Another
+ option is `'mc-intersite-proj'`.
+ w : int
+ Number of random points in the domain for each sample of the
+ training set.
+ """
+
+ def __init__(self, ExpDesign, n_candidate,
+ mc_criterion='mc-intersite-proj-th'):
+ self.ExpDesign = ExpDesign
+ self.n_candidate = n_candidate
+ self.mc_criterion = mc_criterion
+ self.w = 100
+
+ def get_exploration_samples(self):
+ """
+ This function generates candidates to be selected as new design and
+ their associated exploration scores.
+
+ Returns
+ -------
+ all_candidates : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+ explore_method = self.ExpDesign.explore_method
+
+ print("\n")
+ print(f' The {explore_method}-Method is selected as the exploration '
+ 'method.')
+ print("\n")
+
+ if explore_method == 'Voronoi':
+ # Generate samples using the Voronoi method
+ all_candidates, exploration_scores = self.get_vornoi_samples()
+ else:
+ # Generate samples using the MC method
+ all_candidates, exploration_scores = self.get_mc_samples()
+
+ return all_candidates, exploration_scores
+
+ # -------------------------------------------------------------------------
+ def get_vornoi_samples(self):
+ """
+ This function generates samples based on voronoi cells and their
+ corresponding scores
+
+ Returns
+ -------
+ new_samples : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+
+ mc_criterion = self.mc_criterion
+ n_candidate = self.n_candidate
+ # Get the Old ExpDesign #samples
+ old_ED_X = self.ExpDesign.X
+ ndim = old_ED_X.shape[1]
+
+ # calculate error #averageErrors
+ error_voronoi, all_candidates = self.approximate_voronoi(
+ self.w, old_ED_X
+ )
+
+ # Pick the best candidate point in the voronoi cell
+ # for each best sample
+ selected_samples = np.empty((0, ndim))
+ bad_samples = []
+
+ for index in range(len(error_voronoi)):
+
+ # get candidate new samples from voronoi tesselation
+ candidates = self.closest_points[index]
+
+ # get total number of candidates
+ n_new_samples = candidates.shape[0]
+
+ # still no candidate samples around this one, skip it!
+ if n_new_samples == 0:
+ print('The following sample has been skipped because there '
+ 'were no candidate samples around it...')
+ print(old_ED_X[index])
+ bad_samples.append(index)
+ continue
+
+ # find candidate that is farthest away from any existing sample
+ max_min_distance = 0
+ best_candidate = 0
+ min_intersite_dist = np.zeros((n_new_samples))
+ min_projected_dist = np.zeros((n_new_samples))
+
+ for j in range(n_new_samples):
+
+ new_samples = np.vstack((old_ED_X, selected_samples))
+
+ # find min distorted distance from all other samples
+ euclidean_dist = self._build_dist_matrix_point(
+ new_samples, candidates[j], do_sqrt=True)
+ min_euclidean_dist = np.min(euclidean_dist)
+ min_intersite_dist[j] = min_euclidean_dist
+
+ # Check if this is the maximum minimum distance from all other
+ # samples
+ if min_euclidean_dist >= max_min_distance:
+ max_min_distance = min_euclidean_dist
+ best_candidate = j
+
+ # Projected distance
+ projected_dist = distance.cdist(
+ new_samples, [candidates[j]], 'chebyshev')
+ min_projected_dist[j] = np.min(projected_dist)
+
+ if mc_criterion == 'mc-intersite-proj':
+ weight_euclidean_dist = 0.5 * ((n_new_samples+1)**(1/ndim) - 1)
+ weight_projected_dist = 0.5 * (n_new_samples+1)
+ total_dist_scores = weight_euclidean_dist * min_intersite_dist
+ total_dist_scores += weight_projected_dist * min_projected_dist
+
+ elif mc_criterion == 'mc-intersite-proj-th':
+ alpha = 0.5 # chosen (tradeoff)
+ d_min = 2 * alpha / n_new_samples
+ if any(min_projected_dist < d_min):
+ candidates = np.delete(
+ candidates, [min_projected_dist < d_min], axis=0
+ )
+ total_dist_scores = np.delete(
+ min_intersite_dist, [min_projected_dist < d_min],
+ axis=0
+ )
+ else:
+ total_dist_scores = min_intersite_dist
+ else:
+ raise NameError(
+ 'The MC-Criterion you requested is not available.'
+ )
+
+ # Add the best candidate to the list of new samples
+ best_candidate = np.argsort(total_dist_scores)[::-1][:n_candidate]
+ selected_samples = np.vstack(
+ (selected_samples, candidates[best_candidate])
+ )
+
+ self.new_samples = selected_samples
+ self.exploration_scores = np.delete(error_voronoi, bad_samples, axis=0)
+
+ return self.new_samples, self.exploration_scores
+
+ # -------------------------------------------------------------------------
+ def get_mc_samples(self, all_candidates=None):
+ """
+ This function generates random samples based on Global Monte Carlo
+ methods and their corresponding scores, based on [1].
+
+ [1] Crombecq, K., Laermans, E. and Dhaene, T., 2011. Efficient
+ space-filling and non-collapsing sequential design strategies for
+ simulation-based modeling. European Journal of Operational Research
+ , 214(3), pp.683-696.
+ DOI: https://doi.org/10.1016/j.ejor.2011.05.032
+
+ Implemented methods to compute scores:
+ 1) mc-intersite-proj
+ 2) mc-intersite-proj-th
+
+ Arguments
+ ---------
+ all_candidates : array, optional
+ Samples to compute the scores for. The default is `None`. In this
+ case, samples will be generated by defined model input marginals.
+
+ Returns
+ -------
+ new_samples : array of shape (n_candidate, n_params)
+ A list of samples.
+ exploration_scores: arrays of shape (n_candidate)
+ Exploration scores.
+ """
+ explore_method = self.ExpDesign.explore_method
+ mc_criterion = self.mc_criterion
+ if all_candidates is None:
+ n_candidate = self.n_candidate
+ else:
+ n_candidate = all_candidates.shape[0]
+
+ # Get the Old ExpDesign #samples
+ old_ED_X = self.ExpDesign.X
+ ndim = old_ED_X.shape[1]
+
+ # ----- Compute the number of random points -----
+ if all_candidates is None:
+ # Generate MC Samples
+ all_candidates = self.ExpDesign.generate_samples(
+ self.n_candidate, explore_method
+ )
+ self.all_candidates = all_candidates
+
+ # initialization
+ min_intersite_dist = np.zeros((n_candidate))
+ min_projected_dist = np.zeros((n_candidate))
+
+ for i, candidate in enumerate(all_candidates):
+
+ # find candidate that is farthest away from any existing sample
+ maxMinDistance = 0
+
+ # find min distorted distance from all other samples
+ euclidean_dist = self._build_dist_matrix_point(
+ old_ED_X, candidate, do_sqrt=True
+ )
+ min_euclidean_dist = np.min(euclidean_dist)
+ min_intersite_dist[i] = min_euclidean_dist
+
+ # Check if this is the maximum minimum distance from all other
+ # samples
+ if min_euclidean_dist >= maxMinDistance:
+ maxMinDistance = min_euclidean_dist
+
+ # Projected distance
+ projected_dist = self._build_dist_matrix_point(
+ old_ED_X, candidate, 'chebyshev'
+ )
+ min_projected_dist[i] = np.min(projected_dist)
+
+ if mc_criterion == 'mc-intersite-proj':
+ weight_euclidean_dist = ((n_candidate+1)**(1/ndim) - 1) * 0.5
+ weight_projected_dist = (n_candidate+1) * 0.5
+ total_dist_scores = weight_euclidean_dist * min_intersite_dist
+ total_dist_scores += weight_projected_dist * min_projected_dist
+
+ elif mc_criterion == 'mc-intersite-proj-th':
+ alpha = 0.5 # chosen (tradeoff)
+ d_min = 2 * alpha / n_candidate
+ if any(min_projected_dist < d_min):
+ all_candidates = np.delete(
+ all_candidates, [min_projected_dist < d_min], axis=0
+ )
+ total_dist_scores = np.delete(
+ min_intersite_dist, [min_projected_dist < d_min], axis=0
+ )
+ else:
+ total_dist_scores = min_intersite_dist
+ else:
+ raise NameError('The MC-Criterion you requested is not available.')
+
+ self.new_samples = all_candidates
+ self.exploration_scores = total_dist_scores
+ self.exploration_scores /= np.nansum(total_dist_scores)
+
+ return self.new_samples, self.exploration_scores
+
+ # -------------------------------------------------------------------------
+ def approximate_voronoi(self, w, samples):
+ """
+ An approximate (monte carlo) version of Matlab's voronoi command.
+
+ Arguments
+ ---------
+ samples : array
+ Old experimental design to be used as center points for voronoi
+ cells.
+
+ Returns
+ -------
+ areas : array
+ An approximation of the voronoi cells' areas.
+ all_candidates: list of arrays
+ A list of samples in each voronoi cell.
+ """
+ n_samples = samples.shape[0]
+ ndim = samples.shape[1]
+
+ # Compute the number of random points
+ n_points = w * samples.shape[0]
+ # Generate w random points in the domain for each sample
+ points = self.ExpDesign.generate_samples(n_points, 'random')
+ self.all_candidates = points
+
+ # Calculate the nearest sample to each point
+ self.areas = np.zeros((n_samples))
+ self.closest_points = [np.empty((0, ndim)) for i in range(n_samples)]
+
+ # Compute the minimum distance from all the samples of old_ED_X for
+ # each test point
+ for idx in range(n_points):
+ # calculate the minimum distance
+ distances = self._build_dist_matrix_point(
+ samples, points[idx], do_sqrt=True
+ )
+ closest_sample = np.argmin(distances)
+
+ # Add to the voronoi list of the closest sample
+ self.areas[closest_sample] = self.areas[closest_sample] + 1
+ prev_closest_points = self.closest_points[closest_sample]
+ self.closest_points[closest_sample] = np.vstack(
+ (prev_closest_points, points[idx])
+ )
+
+ # Divide by the amount of points to get the estimated volume of each
+ # voronoi cell
+ self.areas /= n_points
+
+ self.perc = np.max(self.areas * 100)
+
+ self.errors = self.areas
+
+ return self.areas, self.all_candidates
+
+ # -------------------------------------------------------------------------
+ def _build_dist_matrix_point(self, samples, point, method='euclidean',
+ do_sqrt=False):
+ """
+ Calculates the intersite distance of all points in samples from point.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params)
+ The old experimental design.
+ point : array
+ A candidate point.
+ method : str
+ Distance method.
+ do_sqrt : bool, optional
+ Whether to return distances or squared distances. The default is
+ `False`.
+
+ Returns
+ -------
+ distances : array
+ Distances.
+
+ """
+ distances = distance.cdist(samples, np.array([point]), method)
+
+ # do square root?
+ if do_sqrt:
+ return distances
+ else:
+ return distances**2
+
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/glexindex.py b/examples/model-comparison/bayesvalidrox/surrogate_models/glexindex.py
new file mode 100644
index 0000000000000000000000000000000000000000..90877331ec121750e7f81e32a4b69edbc9a110ba
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/glexindex.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Multi indices for monomial exponents.
+Credit: Jonathan Feinberg
+https://github.com/jonathf/numpoly/blob/master/numpoly/utils/glexindex.py
+"""
+
+import numpy
+import numpy.typing
+
+
+def glexindex(start, stop=None, dimensions=1, cross_truncation=1.,
+ graded=False, reverse=False):
+ """
+ Generate graded lexicographical multi-indices for the monomial exponents.
+ Args:
+ start (Union[int, numpy.ndarray]):
+ The lower order of the indices. If array of int, counts as lower
+ bound for each axis.
+ stop (Union[int, numpy.ndarray, None]):
+ The maximum shape included. If omitted: stop <- start; start <- 0
+ If int is provided, set as largest total order. If array of int,
+ set as upper bound for each axis.
+ dimensions (int):
+ The number of dimensions in the expansion.
+ cross_truncation (float, Tuple[float, float]):
+ Use hyperbolic cross truncation scheme to reduce the number of
+ terms in expansion. If two values are provided, first is low bound
+ truncation, while the latter upper bound. If only one value, upper
+ bound is assumed.
+ graded (bool):
+ Graded sorting, meaning the indices are always sorted by the index
+ sum. E.g. ``(2, 2, 2)`` has a sum of 6, and will therefore be
+ consider larger than both ``(3, 1, 1)`` and ``(1, 1, 3)``.
+ reverse (bool):
+ Reversed lexicographical sorting meaning that ``(1, 3)`` is
+ considered smaller than ``(3, 1)``, instead of the opposite.
+ Returns:
+ list:
+ Order list of indices.
+ Examples:
+ >>> numpoly.glexindex(4).tolist()
+ [[0], [1], [2], [3]]
+ >>> numpoly.glexindex(2, dimensions=2).tolist()
+ [[0, 0], [1, 0], [0, 1]]
+ >>> numpoly.glexindex(start=2, stop=3, dimensions=2).tolist()
+ [[2, 0], [1, 1], [0, 2]]
+ >>> numpoly.glexindex([1, 2, 3]).tolist()
+ [[0, 0, 0], [0, 1, 0], [0, 0, 1], [0, 0, 2]]
+ >>> numpoly.glexindex([1, 2, 3], cross_truncation=numpy.inf).tolist()
+ [[0, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 1], [0, 0, 2], [0, 1, 2]]
+ """
+ if stop is None:
+ start, stop = 0, start
+ start = numpy.array(start, dtype=int).flatten()
+ stop = numpy.array(stop, dtype=int).flatten()
+ start, stop, _ = numpy.broadcast_arrays(start, stop, numpy.empty(dimensions))
+
+ cross_truncation = cross_truncation*numpy.ones(2)
+
+ # Moved here from _glexindex
+ bound = stop.max()
+ dimensions = len(start)
+ start = numpy.clip(start, a_min=0, a_max=None)
+ dtype = numpy.uint8 if bound < 256 else numpy.uint16
+ range_ = numpy.arange(bound, dtype=dtype)
+ indices = range_[:, numpy.newaxis]
+
+ for idx in range(dimensions-1):
+
+ # Truncate at each step to keep memory usage low
+ if idx:
+ indices = indices[cross_truncate(indices, bound-1, cross_truncation[1])]
+
+ # Repeats the current set of indices.
+ # e.g. [0,1,2] -> [0,1,2,0,1,2,...,0,1,2]
+ indices = numpy.tile(indices, (bound, 1))
+
+ # Stretches ranges over the new dimension.
+ # e.g. [0,1,2] -> [0,0,...,0,1,1,...,1,2,2,...,2]
+ front = range_.repeat(len(indices)//bound)[:, numpy.newaxis]
+
+ # Puts them two together.
+ indices = numpy.column_stack((front, indices))
+
+ # Complete the truncation scheme
+ if dimensions == 1:
+ indices = indices[(indices >= start) & (indices < bound)]
+ else:
+ lower = cross_truncate(indices, start-1, cross_truncation[0])
+ upper = cross_truncate(indices, stop-1, cross_truncation[1])
+ indices = indices[lower ^ upper]
+
+ indices = numpy.array(indices, dtype=int).reshape(-1, dimensions)
+ if indices.size:
+ # moved here from glexsort
+ keys = indices.T
+ keys_ = numpy.atleast_2d(keys)
+ if reverse:
+ keys_ = keys_[::-1]
+
+ indices_sort = numpy.array(numpy.lexsort(keys_))
+ if graded:
+ indices_sort = indices_sort[numpy.argsort(
+ numpy.sum(keys_[:, indices_sort], axis=0))].T
+
+ indices = indices[indices_sort]
+ return indices
+
+def cross_truncate(indices, bound, norm):
+ r"""
+ Truncate of indices using L_p norm.
+ .. math:
+ L_p(x) = \sum_i |x_i/b_i|^p ^{1/p} \leq 1
+ where :math:`b_i` are bounds that each :math:`x_i` should follow.
+ Args:
+ indices (Sequence[int]):
+ Indices to be truncated.
+ bound (int, Sequence[int]):
+ The bound function for witch the indices can not be larger than.
+ norm (float, Sequence[float]):
+ The `p` in the `L_p`-norm. Support includes both `L_0` and `L_inf`.
+ Returns:
+ Boolean indices to ``indices`` with True for each index where the
+ truncation criteria holds.
+ Examples:
+ >>> indices = numpy.array(numpy.mgrid[:10, :10]).reshape(2, -1).T
+ >>> indices[cross_truncate(indices, 2, norm=0)].T
+ array([[0, 0, 0, 1, 2],
+ [0, 1, 2, 0, 0]])
+ >>> indices[cross_truncate(indices, 2, norm=1)].T
+ array([[0, 0, 0, 1, 1, 2],
+ [0, 1, 2, 0, 1, 0]])
+ >>> indices[cross_truncate(indices, [0, 1], norm=1)].T
+ array([[0, 0],
+ [0, 1]])
+ """
+ assert norm >= 0, "negative L_p norm not allowed"
+ bound = numpy.asfarray(bound).flatten()*numpy.ones(indices.shape[1])
+
+ if numpy.any(bound < 0):
+ return numpy.zeros((len(indices),), dtype=bool)
+
+ if numpy.any(bound == 0):
+ out = numpy.all(indices[:, bound == 0] == 0, axis=-1)
+ if numpy.any(bound):
+ out &= cross_truncate(indices[:, bound != 0], bound[bound != 0], norm=norm)
+ return out
+
+ if norm == 0:
+ out = numpy.sum(indices > 0, axis=-1) <= 1
+ out[numpy.any(indices > bound, axis=-1)] = False
+ elif norm == numpy.inf:
+ out = numpy.max(indices/bound, axis=-1) <= 1
+ else:
+ out = numpy.sum((indices/bound)**norm, axis=-1)**(1./norm) <= 1
+
+ assert numpy.all(out[numpy.all(indices == 0, axis=-1)])
+
+ return out
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/input_space.py b/examples/model-comparison/bayesvalidrox/surrogate_models/input_space.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e010d66f2933ec243bad756d8f2c5454808d802
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/input_space.py
@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Input space built from set prior distributions
+"""
+
+import numpy as np
+import chaospy
+import scipy.stats as st
+
+
+class InputSpace:
+ """
+ This class generates the input space for the metamodel from the
+ distributions provided using the `Input` object.
+
+ Attributes
+ ----------
+ Input : obj
+ Input object containing the parameter marginals, i.e. name,
+ distribution type and distribution parameters or available raw data.
+ meta_Model_type : str
+ Type of the meta_Model_type.
+
+ """
+
+ def __init__(self, Input, meta_Model_type='pce'):
+ self.InputObj = Input
+ self.meta_Model_type = meta_Model_type
+
+ # Other
+ self.apce = None
+ self.ndim = None
+
+ # Init
+ self.check_valid_inputs()
+
+
+ def check_valid_inputs(self)-> None:
+ """
+ Check if the given InputObj is valid to use for further calculations:
+ Has some Marginals
+ Marginals have valid priors
+ All Marginals given as the same type (samples vs dist)
+
+ Returns
+ -------
+ None
+
+ """
+ Inputs = self.InputObj
+ self.ndim = len(Inputs.Marginals)
+
+ # Check if PCE or aPCE metamodel is selected.
+ # TODO: test also for 'pce'??
+ if self.meta_Model_type.lower() == 'apce':
+ self.apce = True
+ else:
+ self.apce = False
+
+ # check if marginals given
+ if not self.ndim >=1:
+ raise AssertionError('Cannot build distributions if no marginals are given')
+
+ # check that each marginal is valid
+ for marginals in Inputs.Marginals:
+ if len(marginals.input_data) == 0:
+ if marginals.dist_type == None:
+ raise AssertionError('Not all marginals were provided priors')
+ break
+ if np.array(marginals.input_data).shape[0] and (marginals.dist_type != None):
+ raise AssertionError('Both samples and distribution type are given. Please choose only one.')
+ break
+
+ # Check if input is given as dist or input_data.
+ self.input_data_given = -1
+ for marg in Inputs.Marginals:
+ #print(self.input_data_given)
+ size = np.array(marg.input_data).shape[0]
+ #print(f'Size: {size}')
+ if size and abs(self.input_data_given) !=1:
+ self.input_data_given = 2
+ break
+ if (not size) and self.input_data_given > 0:
+ self.input_data_given = 2
+ break
+ if not size:
+ self.input_data_given = 0
+ if size:
+ self.input_data_given = 1
+
+ if self.input_data_given == 2:
+ raise AssertionError('Distributions cannot be built as the priors have different types')
+
+
+ # Get the bounds if input_data are directly defined by user:
+ if self.input_data_given:
+ for i in range(self.ndim):
+ low_bound = np.min(Inputs.Marginals[i].input_data)
+ up_bound = np.max(Inputs.Marginals[i].input_data)
+ Inputs.Marginals[i].parameters = [low_bound, up_bound]
+
+
+
+ # -------------------------------------------------------------------------
+ def init_param_space(self, max_deg=None):
+ """
+ Initializes parameter space.
+
+ Parameters
+ ----------
+ max_deg : int, optional
+ Maximum degree. The default is `None`.
+
+ Creates
+ -------
+ raw_data : array of shape (n_params, n_samples)
+ Raw data.
+ bound_tuples : list of tuples
+ A list containing lower and upper bounds of parameters.
+
+ """
+ # Recheck all before running!
+ self.check_valid_inputs()
+
+ Inputs = self.InputObj
+ ndim = self.ndim
+ rosenblatt_flag = Inputs.Rosenblatt
+ mc_size = 50000
+
+ # Save parameter names
+ self.par_names = []
+ for parIdx in range(ndim):
+ self.par_names.append(Inputs.Marginals[parIdx].name)
+
+ # Create a multivariate probability distribution
+ # TODO: change this to make max_deg obligatory? at least in some specific cases?
+ if max_deg is not None:
+ JDist, poly_types = self.build_polytypes(rosenblatt=rosenblatt_flag)
+ self.JDist, self.poly_types = JDist, poly_types
+
+ if self.input_data_given:
+ self.MCSize = len(Inputs.Marginals[0].input_data)
+ self.raw_data = np.zeros((ndim, self.MCSize))
+
+ for parIdx in range(ndim):
+ # Save parameter names
+ try:
+ self.raw_data[parIdx] = np.array(
+ Inputs.Marginals[parIdx].input_data)
+ except:
+ self.raw_data[parIdx] = self.JDist[parIdx].sample(mc_size)
+
+ else:
+ # Generate random samples based on parameter distributions
+ self.raw_data = chaospy.generate_samples(mc_size,
+ domain=self.JDist)
+
+ # Extract moments
+ for parIdx in range(ndim):
+ mu = np.mean(self.raw_data[parIdx])
+ std = np.std(self.raw_data[parIdx])
+ self.InputObj.Marginals[parIdx].moments = [mu, std]
+
+ # Generate the bounds based on given inputs for marginals
+ bound_tuples = []
+ for i in range(ndim):
+ if Inputs.Marginals[i].dist_type == 'unif':
+ low_bound = Inputs.Marginals[i].parameters[0]
+ up_bound = Inputs.Marginals[i].parameters[1]
+ else:
+ low_bound = np.min(self.raw_data[i])
+ up_bound = np.max(self.raw_data[i])
+
+ bound_tuples.append((low_bound, up_bound))
+
+ self.bound_tuples = tuple(bound_tuples)
+
+ # -------------------------------------------------------------------------
+ def build_polytypes(self, rosenblatt):
+ """
+ Creates the polynomial types to be passed to univ_basis_vals method of
+ the MetaModel object.
+
+ Parameters
+ ----------
+ rosenblatt : bool
+ Rosenblatt transformation flag.
+
+ Returns
+ -------
+ orig_space_dist : object
+ A chaospy JDist object or a gaussian_kde object.
+ poly_types : list
+ List of polynomial types for the parameters.
+
+ """
+ Inputs = self.InputObj
+
+ all_data = []
+ all_dist_types = []
+ orig_joints = []
+ poly_types = []
+
+ for parIdx in range(self.ndim):
+
+ if Inputs.Marginals[parIdx].dist_type is None:
+ data = Inputs.Marginals[parIdx].input_data
+ all_data.append(data)
+ dist_type = None
+ else:
+ dist_type = Inputs.Marginals[parIdx].dist_type
+ params = Inputs.Marginals[parIdx].parameters
+
+ if rosenblatt:
+ polytype = 'hermite'
+ dist = chaospy.Normal()
+
+ elif dist_type is None:
+ polytype = 'arbitrary'
+ dist = None
+
+ elif 'unif' in dist_type.lower():
+ polytype = 'legendre'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Uniform(lower=params[0], upper=params[1])
+
+ elif 'norm' in dist_type.lower() and \
+ 'log' not in dist_type.lower():
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ polytype = 'hermite'
+ dist = chaospy.Normal(mu=params[0], sigma=params[1])
+
+ elif 'gamma' in dist_type.lower():
+ polytype = 'laguerre'
+ if not np.array(params).shape[0]>=3:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Gamma(shape=params[0],
+ scale=params[1],
+ shift=params[2])
+
+ elif 'beta' in dist_type.lower():
+ if not np.array(params).shape[0]>=4:
+ raise AssertionError('Distribution has too few parameters!')
+ polytype = 'jacobi'
+ dist = chaospy.Beta(alpha=params[0], beta=params[1],
+ lower=params[2], upper=params[3])
+
+ elif 'lognorm' in dist_type.lower():
+ polytype = 'hermite'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ mu = np.log(params[0]**2/np.sqrt(params[0]**2 + params[1]**2))
+ sigma = np.sqrt(np.log(1 + params[1]**2 / params[0]**2))
+ dist = chaospy.LogNormal(mu, sigma)
+ # dist = chaospy.LogNormal(mu=params[0], sigma=params[1])
+
+ elif 'expon' in dist_type.lower():
+ polytype = 'exponential'
+ if not np.array(params).shape[0]>=2:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Exponential(scale=params[0], shift=params[1])
+
+ elif 'weibull' in dist_type.lower():
+ polytype = 'weibull'
+ if not np.array(params).shape[0]>=3:
+ raise AssertionError('Distribution has too few parameters!')
+ dist = chaospy.Weibull(shape=params[0], scale=params[1],
+ shift=params[2])
+
+ else:
+ message = (f"DistType {dist_type} for parameter"
+ f"{parIdx+1} is not available.")
+ raise ValueError(message)
+
+ if self.input_data_given or self.apce:
+ polytype = 'arbitrary'
+
+ # Store dists and poly_types
+ orig_joints.append(dist)
+ poly_types.append(polytype)
+ all_dist_types.append(dist_type)
+
+ # Prepare final output to return
+ if None in all_dist_types:
+ # Naive approach: Fit a gaussian kernel to the provided data
+ Data = np.asarray(all_data)
+ try:
+ orig_space_dist = st.gaussian_kde(Data)
+ except:
+ raise ValueError('The samples provided to the Marginals should be 1D only')
+ self.prior_space = orig_space_dist
+ else:
+ orig_space_dist = chaospy.J(*orig_joints)
+ try:
+ self.prior_space = st.gaussian_kde(orig_space_dist.sample(10000))
+ except:
+ raise ValueError('Parameter values are not valid, please set differently')
+
+ return orig_space_dist, poly_types
+
+ # -------------------------------------------------------------------------
+ def transform(self, X, params=None, method=None):
+ """
+ Transforms the samples via either a Rosenblatt or an isoprobabilistic
+ transformation.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples,n_params)
+ Samples to be transformed.
+ method : string
+ If transformation method is 'user' transform X, else just pass X.
+
+ Returns
+ -------
+ tr_X: array of shape (n_samples,n_params)
+ Transformed samples.
+
+ """
+ # Check for built JDist
+ if not hasattr(self, 'JDist'):
+ raise AttributeError('Call function init_param_space first to create JDist')
+
+ # Check if X is 2d
+ if X.ndim != 2:
+ raise AttributeError('X should have two dimensions')
+
+ # Check if size of X matches Marginals
+ if X.shape[1]!= self.ndim:
+ raise AttributeError('The second dimension of X should be the same size as the number of marginals in the InputObj')
+
+ if self.InputObj.Rosenblatt:
+ self.origJDist, _ = self.build_polytypes(False)
+ if method == 'user':
+ tr_X = self.JDist.inv(self.origJDist.fwd(X.T)).T
+ else:
+ # Inverse to original spcace -- generate sample ED
+ tr_X = self.origJDist.inv(self.JDist.fwd(X.T)).T
+ else:
+ # Transform samples via an isoprobabilistic transformation
+ n_samples, n_params = X.shape
+ Inputs = self.InputObj
+ origJDist = self.JDist
+ poly_types = self.poly_types
+
+ disttypes = []
+ for par_i in range(n_params):
+ disttypes.append(Inputs.Marginals[par_i].dist_type)
+
+ # Pass non-transformed X, if arbitrary PCE is selected.
+ if None in disttypes or self.input_data_given or self.apce:
+ return X
+
+ cdfx = np.zeros((X.shape))
+ tr_X = np.zeros((X.shape))
+
+ for par_i in range(n_params):
+
+ # Extract the parameters of the original space
+ disttype = disttypes[par_i]
+ if disttype is not None:
+ dist = origJDist[par_i]
+ else:
+ dist = None
+ polytype = poly_types[par_i]
+ cdf = np.vectorize(lambda x: dist.cdf(x))
+
+ # Extract the parameters of the transformation space based on
+ # polyType
+ if polytype == 'legendre' or disttype == 'uniform':
+ # Generate Y_Dists based
+ params_Y = [-1, 1]
+ dist_Y = st.uniform(loc=params_Y[0],
+ scale=params_Y[1]-params_Y[0])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ elif polytype == 'hermite' or disttype == 'norm':
+ params_Y = [0, 1]
+ dist_Y = st.norm(loc=params_Y[0], scale=params_Y[1])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ elif polytype == 'laguerre' or disttype == 'gamma':
+ if params == None:
+ raise AttributeError('Additional parameters have to be set for the gamma distribution!')
+ params_Y = [1, params[1]]
+ dist_Y = st.gamma(loc=params_Y[0], scale=params_Y[1])
+ inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
+
+ # Compute CDF_x(X)
+ cdfx[:, par_i] = cdf(X[:, par_i])
+
+ # Compute invCDF_y(cdfx)
+ tr_X[:, par_i] = inv_cdf(cdfx[:, par_i])
+
+ return tr_X
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/inputs.py b/examples/model-comparison/bayesvalidrox/surrogate_models/inputs.py
new file mode 100644
index 0000000000000000000000000000000000000000..094e1066fe008e37288e44750524c5a1370bd7a2
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/inputs.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Inputs and related marginal distributions
+"""
+
+class Input:
+ """
+ A class to define the uncertain input parameters.
+
+ Attributes
+ ----------
+ Marginals : obj
+ Marginal objects. See `inputs.Marginal`.
+ Rosenblatt : bool
+ If Rossenblatt transformation is required for the dependent input
+ parameters.
+
+ Examples
+ -------
+ Marginals can be defined as following:
+
+ >>> Inputs.add_marginals()
+ >>> Inputs.Marginals[0].name = 'X_1'
+ >>> Inputs.Marginals[0].dist_type = 'uniform'
+ >>> Inputs.Marginals[0].parameters = [-5, 5]
+
+ If there is no common data is avaliable, the input data can be given
+ as following:
+
+ >>> Inputs.add_marginals()
+ >>> Inputs.Marginals[0].name = 'X_1'
+ >>> Inputs.Marginals[0].input_data = input_data
+ """
+ poly_coeffs_flag = True
+
+ def __init__(self):
+ self.Marginals = []
+ self.Rosenblatt = False
+
+ def add_marginals(self):
+ """
+ Adds a new Marginal object to the input object.
+
+ Returns
+ -------
+ None.
+
+ """
+ self.Marginals.append(Marginal())
+
+
+# Nested class
+class Marginal:
+ """
+ An object containing the specifications of the marginals for each uncertain
+ parameter.
+
+ Attributes
+ ----------
+ name : string
+ Name of the parameter. The default is `'$x_1$'`.
+ dist_type : string
+ Name of the distribution. The default is `None`.
+ parameters : list
+ List of the parameters corresponding to the distribution type. The
+ default is `None`.
+ input_data : array
+ Available input data. The default is `[]`.
+ moments : list
+ List of the moments.
+ """
+
+ def __init__(self):
+ self.name = '$x_1$'
+ self.dist_type = None
+ self.parameters = None
+ self.input_data = []
+ self.moments = None
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py b/examples/model-comparison/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py
new file mode 100644
index 0000000000000000000000000000000000000000..96ef9c1d50b10b587ad0846d41733fc7f1cedfe8
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py
@@ -0,0 +1,366 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jul 15 14:08:59 2022
+
+@author: farid
+"""
+import numpy as np
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+from sklearn.utils import check_X_y
+
+
+def corr(x, y):
+ return abs(x.dot(y))/np.sqrt((x**2).sum())
+
+
+class OrthogonalMatchingPursuit(LinearModel, RegressorMixin):
+ '''
+ Regression with Orthogonal Matching Pursuit [1].
+
+ Parameters
+ ----------
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ References
+ ----------
+ [1] Pati, Y., Rezaiifar, R., Krishnaprasad, P. (1993). Orthogonal matching
+ pursuit: recursive function approximation with application to wavelet
+ decomposition. Proceedings of 27th Asilomar Conference on Signals,
+ Systems and Computers, 40-44.
+ '''
+
+ def __init__(self, fit_intercept=True, normalize=False, copy_X=True,
+ verbose=False):
+ self.fit_intercept = fit_intercept
+ self.normalize = normalize
+ self.copy_X = copy_X
+ self.verbose = verbose
+
+ def _preprocess_data(self, X, y):
+ """Center and scale data.
+ Centers data to have mean zero along axis 0. If fit_intercept=False or
+ if the X is a sparse matrix, no centering is done, but normalization
+ can still be applied. The function returns the statistics necessary to
+ reconstruct the input data, which are X_offset, y_offset, X_scale, such
+ that the output
+ X = (X - X_offset) / X_scale
+ X_scale is the L2 norm of X - X_offset.
+ """
+
+ if self.copy_X:
+ X = X.copy(order='K')
+
+ y = np.asarray(y, dtype=X.dtype)
+
+ if self.fit_intercept:
+ X_offset = np.average(X, axis=0)
+ X -= X_offset
+ if self.normalize:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ std = np.sqrt(np.sum(X**2, axis=0)/(len(X)-1))
+ X_scale[std != 0] = std[std != 0]
+ X /= X_scale
+ else:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ y_offset = np.mean(y)
+ y = y - y_offset
+ else:
+ X_offset = np.zeros(X.shape[1], dtype=X.dtype)
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ if y.ndim == 1:
+ y_offset = X.dtype.type(0)
+ else:
+ y_offset = np.zeros(y.shape[1], dtype=X.dtype)
+
+ return X, y, X_offset, y_offset, X_scale
+
+ def fit(self, X, y):
+ '''
+ Fits Regression with Orthogonal Matching Pursuit Algorithm.
+
+ Parameters
+ -----------
+ X: {array-like, sparse matrix} of size (n_samples, n_features)
+ Training data, matrix of explanatory variables
+
+ y: array-like of size [n_samples, n_features]
+ Target values
+
+ Returns
+ -------
+ self : object
+ Returns self.
+ '''
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._preprocess_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # Normalize columns of Psi, so that each column has norm = 1
+ norm_X = np.linalg.norm(X, axis=0)
+ X_norm = X/norm_X
+
+ # Initialize residual vector to full model response and normalize
+ R = y
+ norm_y = np.sqrt(np.dot(y, y))
+ r = y/norm_y
+
+ # Check for constant regressors
+ const_indices = np.where(~np.diff(X, axis=0).any(axis=0))[0]
+ bool_const = not const_indices
+
+ # Start regression using OPM algorithm
+ precision = 0 # Set precision criterion to precision of program
+ early_stop = True
+ cond_early = True # Initialize condition for early stop
+ ind = []
+ iindx = [] # index of selected columns
+ indtot = np.arange(n_features) # Full index set for remaining columns
+ kmax = min(n_samples, n_features) # Maximum number of iterations
+ LOO = np.PINF * np.ones(kmax) # Store LOO error at each iteration
+ LOOmin = np.PINF # Initialize minimum value of LOO
+ coeff = np.zeros((n_features, kmax))
+ count = 0
+ k = 0.1 # Percentage of iteration history for early stop
+
+ # Begin iteration over regressors set (Matrix X)
+ while (np.linalg.norm(R) > precision) and (count <= kmax-1) and \
+ ((cond_early or early_stop) ^ ~cond_early):
+
+ # Update index set of columns yet to select
+ if count != 0:
+ indtot = np.delete(indtot, iindx)
+
+ # Find column of X that is most correlated with residual
+ h = abs(np.dot(r, X_norm))
+ iindx = np.argmax(h[indtot])
+ indx = indtot[iindx]
+
+ # initialize with the constant regressor, if it exists in the basis
+ if (count == 0) and bool_const:
+ # overwrite values for iindx and indx
+ iindx = const_indices[0]
+ indx = indtot[iindx]
+
+ # Invert the information matrix at the first iteration, later only
+ # update its value on the basis of the previously inverted one,
+ if count == 0:
+ M = 1 / np.dot(X[:, indx], X[:, indx])
+ else:
+ x = np.dot(X[:, ind].T, X[:, indx])
+ r = np.dot(X[:, indx], X[:, indx])
+ M = self.blockwise_inverse(M, x, x.T, r)
+
+ # Add newly found index to the selected indexes set
+ ind.append(indx)
+
+ # Select regressors subset (Projection subspace)
+ Xpro = X[:, ind]
+
+ # Obtain coefficient by performing OLS
+ TT = np.dot(y, Xpro)
+ beta = np.dot(M, TT)
+ coeff[ind, count] = beta
+
+ # Compute LOO error
+ LOO[count] = self.loo_error(Xpro, M, y, beta)
+
+ # Compute new residual due to new projection
+ R = y - np.dot(Xpro, beta)
+
+ # Normalize residual
+ norm_R = np.sqrt(np.dot(R, R))
+ r = R / norm_R
+
+ # Update counters and early-stop criterions
+ countinf = max(0, int(count-k*kmax))
+ LOOmin = min(LOOmin, LOO[count])
+
+ if count == 0:
+ cond_early = (LOO[0] <= LOOmin)
+ else:
+ cond_early = (min(LOO[countinf:count+1]) <= LOOmin)
+
+ if self.verbose:
+ print(f'Iteration: {count+1}, mod. LOOCV error : '
+ f'{LOO[count]:.2e}')
+
+ # Update counter
+ count += 1
+
+ # Select projection with smallest cross-validation error
+ countmin = np.argmin(LOO[:-1])
+ self.coef_ = coeff[:, countmin]
+ self.active = coeff[:, countmin] != 0.0
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self, X):
+ '''
+ Computes predictive distribution for test set.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+ '''
+
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ return y_hat
+
+ def loo_error(self, psi, inv_inf_matrix, y, coeffs):
+ """
+ Calculates the corrected LOO error for regression on regressor
+ matrix `psi` that generated the coefficients based on [1] and [2].
+
+ [1] Blatman, G., 2009. Adaptive sparse polynomial chaos expansions for
+ uncertainty propagation and sensitivity analysis (Doctoral
+ dissertation, Clermont-Ferrand 2).
+
+ [2] Blatman, G. and Sudret, B., 2011. Adaptive sparse polynomial chaos
+ expansion based on least angle regression. Journal of computational
+ Physics, 230(6), pp.2345-2367.
+
+ Parameters
+ ----------
+ psi : array of shape (n_samples, n_feature)
+ Orthogonal bases evaluated at the samples.
+ inv_inf_matrix : array
+ Inverse of the information matrix.
+ y : array of shape (n_samples, )
+ Targets.
+ coeffs : array
+ Computed regresssor cofficients.
+
+ Returns
+ -------
+ loo_error : float
+ Modified LOOCV error.
+
+ """
+
+ # NrEvaluation (Size of experimental design)
+ N, P = psi.shape
+
+ # h factor (the full matrix is not calculated explicitly,
+ # only the trace is, to save memory)
+ PsiM = np.dot(psi, inv_inf_matrix)
+
+ h = np.sum(np.multiply(PsiM, psi), axis=1, dtype=np.longdouble)
+
+ # ------ Calculate Error Loocv for each measurement point ----
+ # Residuals
+ residual = np.dot(psi, coeffs) - y
+
+ # Variance
+ varY = np.var(y)
+
+ if varY == 0:
+ norm_emp_error = 0
+ loo_error = 0
+ else:
+ norm_emp_error = np.mean(residual**2)/varY
+
+ loo_error = np.mean(np.square(residual / (1-h))) / varY
+
+ # if there are NaNs, just return an infinite LOO error (this
+ # happens, e.g., when a strongly underdetermined problem is solved)
+ if np.isnan(loo_error):
+ loo_error = np.inf
+
+ # Corrected Error for over-determined system
+ tr_M = np.trace(np.atleast_2d(inv_inf_matrix))
+ if tr_M < 0 or abs(tr_M) > 1e6:
+ tr_M = np.trace(np.linalg.pinv(np.dot(psi.T, psi)))
+
+ # Over-determined system of Equation
+ if N > P:
+ T_factor = N/(N-P) * (1 + tr_M)
+
+ # Under-determined system of Equation
+ else:
+ T_factor = np.inf
+
+ loo_error *= T_factor
+
+ return loo_error
+
+ def blockwise_inverse(self, Ainv, B, C, D):
+ """
+ non-singular square matrix M defined as M = [[A B]; [C D]] .
+ B, C and D can have any dimension, provided their combination defines
+ a square matrix M.
+
+ Parameters
+ ----------
+ Ainv : float or array
+ inverse of the square-submatrix A.
+ B : float or array
+ Information matrix with all new regressor.
+ C : float or array
+ Transpose of B.
+ D : float or array
+ Information matrix with all selected regressors.
+
+ Returns
+ -------
+ M : array
+ Inverse of the information matrix.
+
+ """
+ if np.isscalar(D):
+ # Inverse of D
+ Dinv = 1/D
+ # Schur complement
+ SCinv = 1/(D - np.dot(C, np.dot(Ainv, B[:, None])))[0]
+ else:
+ # Inverse of D
+ Dinv = np.linalg.solve(D, np.eye(D.shape))
+ # Schur complement
+ SCinv = np.linalg.solve((D - C*Ainv*B), np.eye(D.shape))
+
+ T1 = np.dot(Ainv, np.dot(B[:, None], SCinv))
+ T2 = np.dot(C, Ainv)
+
+ # Assemble the inverse matrix
+ M = np.vstack((
+ np.hstack((Ainv+T1*T2, -T1)),
+ np.hstack((-(SCinv)*T2, SCinv))
+ ))
+ return M
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_ard.py b/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_ard.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6883a3edd6d247c219b8be328f5206b75780fbb
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_ard.py
@@ -0,0 +1,475 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar 24 19:41:45 2020
+
+@author: farid
+"""
+import numpy as np
+from scipy.linalg import solve_triangular
+from numpy.linalg import LinAlgError
+from sklearn.base import RegressorMixin
+from sklearn.linear_model._base import LinearModel
+import warnings
+from sklearn.utils import check_X_y
+from scipy.linalg import pinvh
+
+
+def update_precisions(Q,S,q,s,A,active,tol,n_samples,clf_bias):
+ '''
+ Selects one feature to be added/recomputed/deleted to model based on
+ effect it will have on value of log marginal likelihood.
+ '''
+ # initialise vector holding changes in log marginal likelihood
+ deltaL = np.zeros(Q.shape[0])
+
+ # identify features that can be added , recomputed and deleted in model
+ theta = q**2 - s
+ add = (theta > 0) * (active == False)
+ recompute = (theta > 0) * (active == True)
+ delete = ~(add + recompute)
+
+ # compute sparsity & quality parameters corresponding to features in
+ # three groups identified above
+ Qadd,Sadd = Q[add], S[add]
+ Qrec,Srec,Arec = Q[recompute], S[recompute], A[recompute]
+ Qdel,Sdel,Adel = Q[delete], S[delete], A[delete]
+
+ # compute new alpha's (precision parameters) for features that are
+ # currently in model and will be recomputed
+ Anew = s[recompute]**2/ ( theta[recompute] + np.finfo(np.float32).eps)
+ delta_alpha = (1./Anew - 1./Arec)
+
+ # compute change in log marginal likelihood
+ deltaL[add] = ( Qadd**2 - Sadd ) / Sadd + np.log(Sadd/Qadd**2 )
+ deltaL[recompute] = Qrec**2 / (Srec + 1. / delta_alpha) - np.log(1 + Srec*delta_alpha)
+ deltaL[delete] = Qdel**2 / (Sdel - Adel) - np.log(1 - Sdel / Adel)
+ deltaL = deltaL / n_samples
+
+ # find feature which caused largest change in likelihood
+ feature_index = np.argmax(deltaL)
+
+ # no deletions or additions
+ same_features = np.sum( theta[~recompute] > 0) == 0
+
+ # changes in precision for features already in model is below threshold
+ no_delta = np.sum( abs( Anew - Arec ) > tol ) == 0
+ # if same_features: print(abs( Anew - Arec ))
+ # print("same_features = {} no_delta = {}".format(same_features,no_delta))
+ # check convergence: if no features to add or delete and small change in
+ # precision for current features then terminate
+ converged = False
+ if same_features and no_delta:
+ converged = True
+ return [A,converged]
+
+ # if not converged update precision parameter of weights and return
+ if theta[feature_index] > 0:
+ A[feature_index] = s[feature_index]**2 / theta[feature_index]
+ if active[feature_index] == False:
+ active[feature_index] = True
+ else:
+ # at least two active features
+ if active[feature_index] == True and np.sum(active) >= 2:
+ # do not remove bias term in classification
+ # (in regression it is factored in through centering)
+ if not (feature_index == 0 and clf_bias):
+ active[feature_index] = False
+ A[feature_index] = np.PINF
+
+ return [A,converged]
+
+
+class RegressionFastARD(LinearModel, RegressorMixin):
+ '''
+ Regression with Automatic Relevance Determination (Fast Version uses
+ Sparse Bayesian Learning)
+ https://github.com/AmazaspShumik/sklearn-bayes/blob/master/skbayes/rvm_ard_models/fast_rvm.py
+
+ Parameters
+ ----------
+ n_iter: int, optional (DEFAULT = 100)
+ Maximum number of iterations
+
+ start: list, optional (DEFAULT = None)
+ Initial selected features.
+
+ tol: float, optional (DEFAULT = 1e-3)
+ If absolute change in precision parameter for weights is below threshold
+ algorithm terminates.
+
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ compute_score : bool, default=False
+ If True, compute the log marginal likelihood at each iteration of the
+ optimization.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ alpha_ : float
+ estimated precision of the noise
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ lambda_ : array, shape = (n_features)
+ estimated precisions of the coefficients
+
+ sigma_ : array, shape = (n_features, n_features)
+ estimated covariance matrix of the weights, computed only
+ for non-zero coefficients
+
+ scores_ : array-like of shape (n_iter_+1,)
+ If computed_score is True, value of the log marginal likelihood (to be
+ maximized) at each iteration of the optimization.
+
+ References
+ ----------
+ [1] Fast marginal likelihood maximisation for sparse Bayesian models
+ (Tipping & Faul 2003) (http://www.miketipping.com/papers/met-fastsbl.pdf)
+ [2] Analysis of sparse Bayesian learning (Tipping & Faul 2001)
+ (http://www.miketipping.com/abstracts.htm#Faul:NIPS01)
+ '''
+
+ def __init__(self, n_iter=300, start=None, tol=1e-3, fit_intercept=True,
+ normalize=False, copy_X=True, compute_score=False, verbose=False):
+ self.n_iter = n_iter
+ self.start = start
+ self.tol = tol
+ self.scores_ = list()
+ self.fit_intercept = fit_intercept
+ self.normalize = normalize
+ self.copy_X = copy_X
+ self.compute_score = compute_score
+ self.verbose = verbose
+
+ def _preprocess_data(self, X, y):
+ """Center and scale data.
+ Centers data to have mean zero along axis 0. If fit_intercept=False or
+ if the X is a sparse matrix, no centering is done, but normalization
+ can still be applied. The function returns the statistics necessary to
+ reconstruct the input data, which are X_offset, y_offset, X_scale, such
+ that the output
+ X = (X - X_offset) / X_scale
+ X_scale is the L2 norm of X - X_offset.
+ """
+
+ if self.copy_X:
+ X = X.copy(order='K')
+
+ y = np.asarray(y, dtype=X.dtype)
+
+ if self.fit_intercept:
+ X_offset = np.average(X, axis=0)
+ X -= X_offset
+ if self.normalize:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ std = np.sqrt(np.sum(X**2, axis=0)/(len(X)-1))
+ X_scale[std != 0] = std[std != 0]
+ X /= X_scale
+ else:
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ y_offset = np.mean(y)
+ y = y - y_offset
+ else:
+ X_offset = np.zeros(X.shape[1], dtype=X.dtype)
+ X_scale = np.ones(X.shape[1], dtype=X.dtype)
+ if y.ndim == 1:
+ y_offset = X.dtype.type(0)
+ else:
+ y_offset = np.zeros(y.shape[1], dtype=X.dtype)
+
+ return X, y, X_offset, y_offset, X_scale
+
+ def fit(self, X, y):
+ '''
+ Fits ARD Regression with Sequential Sparse Bayes Algorithm.
+
+ Parameters
+ -----------
+ X: {array-like, sparse matrix} of size (n_samples, n_features)
+ Training data, matrix of explanatory variables
+
+ y: array-like of size [n_samples, n_features]
+ Target values
+
+ Returns
+ -------
+ self : object
+ Returns self.
+ '''
+ X, y = check_X_y(X, y, dtype=np.float64, y_numeric=True)
+ n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._preprocess_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # precompute X'*Y , X'*X for faster iterations & allocate memory for
+ # sparsity & quality vectors
+ XY = np.dot(X.T, y)
+ XX = np.dot(X.T, X)
+ XXd = np.diag(XX)
+
+ # initialise precision of noise & and coefficients
+ var_y = np.var(y)
+
+ # check that variance is non zero !!!
+ if var_y == 0:
+ beta = 1e-2
+ self.var_y = True
+ else:
+ beta = 1. / np.var(y)
+ self.var_y = False
+
+ A = np.PINF * np.ones(n_features)
+ active = np.zeros(n_features, dtype=np.bool)
+
+ if self.start is not None and not hasattr(self, 'active_'):
+ start = self.start
+ # start from a given start basis vector
+ proj = XY**2 / XXd
+ active[start] = True
+ A[start] = XXd[start]/(proj[start] - var_y)
+
+ else:
+ # in case of almost perfect multicollinearity between some features
+ # start from feature 0
+ if np.sum(XXd - X_mean**2 < np.finfo(np.float32).eps) > 0:
+ A[0] = np.finfo(np.float16).eps
+ active[0] = True
+
+ else:
+ # start from a single basis vector with largest projection on
+ # targets
+ proj = XY**2 / XXd
+ start = np.argmax(proj)
+ active[start] = True
+ A[start] = XXd[start]/(proj[start] - var_y +
+ np.finfo(np.float32).eps)
+
+ warning_flag = 0
+ scores_ = []
+ for i in range(self.n_iter):
+ # Handle variance zero
+ if self.var_y:
+ A[0] = y_mean
+ active[0] = True
+ converged = True
+ break
+
+ XXa = XX[active, :][:, active]
+ XYa = XY[active]
+ Aa = A[active]
+
+ # mean & covariance of posterior distribution
+ Mn, Ri, cholesky = self._posterior_dist(Aa, beta, XXa, XYa)
+ if cholesky:
+ Sdiag = np.sum(Ri**2, 0)
+ else:
+ Sdiag = np.copy(np.diag(Ri))
+ warning_flag += 1
+
+ # raise warning in case cholesky fails
+ if warning_flag == 1:
+ warnings.warn(("Cholesky decomposition failed! Algorithm uses "
+ "pinvh, which is significantly slower. If you "
+ "use RVR it is advised to change parameters of "
+ "the kernel!"))
+
+ # compute quality & sparsity parameters
+ s, q, S, Q = self._sparsity_quality(XX, XXd, XY, XYa, Aa, Ri,
+ active, beta, cholesky)
+
+ # update precision parameter for noise distribution
+ rss = np.sum((y - np.dot(X[:, active], Mn))**2)
+
+ # if near perfect fit , then terminate
+ if (rss / n_samples/var_y) < self.tol:
+ warnings.warn('Early termination due to near perfect fit')
+ converged = True
+ break
+ beta = n_samples - np.sum(active) + np.sum(Aa * Sdiag)
+ beta /= rss
+ # beta /= (rss + np.finfo(np.float32).eps)
+
+ # update precision parameters of coefficients
+ A, converged = update_precisions(Q, S, q, s, A, active, self.tol,
+ n_samples, False)
+
+ if self.compute_score:
+ scores_.append(self.log_marginal_like(XXa, XYa, Aa, beta))
+
+ if self.verbose:
+ print(('Iteration: {0}, number of features '
+ 'in the model: {1}').format(i, np.sum(active)))
+
+ if converged or i == self.n_iter - 1:
+ if converged and self.verbose:
+ print('Algorithm converged!')
+ break
+
+ # after last update of alpha & beta update parameters
+ # of posterior distribution
+ XXa, XYa, Aa = XX[active, :][:, active], XY[active], A[active]
+ Mn, Sn, cholesky = self._posterior_dist(Aa, beta, XXa, XYa, True)
+ self.coef_ = np.zeros(n_features)
+ self.coef_[active] = Mn
+ self.sigma_ = Sn
+ self.active_ = active
+ self.lambda_ = A
+ self.alpha_ = beta
+ self.converged = converged
+ if self.compute_score:
+ self.scores_ = np.array(scores_)
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+ return self
+
+ def log_marginal_like(self, XXa, XYa, Aa, beta):
+ """Computes the log of the marginal likelihood."""
+ N, M = XXa.shape
+ A = np.diag(Aa)
+
+ Mn, sigma_, cholesky = self._posterior_dist(Aa, beta, XXa, XYa,
+ full_covar=True)
+
+ C = sigma_ + np.dot(np.dot(XXa.T, np.linalg.pinv(A)), XXa)
+
+ score = np.dot(np.dot(XYa.T, np.linalg.pinv(C)), XYa) +\
+ np.log(np.linalg.det(C)) + N * np.log(2 * np.pi)
+
+ return -0.5 * score
+
+ def predict(self, X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance based on
+ Ref.[1] Section 3.3.2.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+ References
+ ----------
+ [1] Bishop, C. M. (2006). Pattern recognition and machine learning.
+ springer.
+ '''
+
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ if return_std:
+ # Handle the zero variance case
+ if self.var_y:
+ return y_hat, np.zeros_like(y_hat)
+
+ if self.normalize:
+ X -= self._x_mean_[self.active_]
+ X /= self._x_std[self.active_]
+ var_hat = 1./self.alpha_
+ var_hat += np.sum(X.dot(self.sigma_) * X, axis=1)
+ std_hat = np.sqrt(var_hat)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+ def _posterior_dist(self, A, beta, XX, XY, full_covar=False):
+ '''
+ Calculates mean and covariance matrix of posterior distribution
+ of coefficients.
+ '''
+ # compute precision matrix for active features
+ Sinv = beta * XX
+ np.fill_diagonal(Sinv, np.diag(Sinv) + A)
+ cholesky = True
+
+ # try cholesky, if it fails go back to pinvh
+ try:
+ # find posterior mean : R*R.T*mean = beta*X.T*Y
+ # solve(R*z = beta*X.T*Y) =>find z=> solve(R.T*mean = z)=>find mean
+ R = np.linalg.cholesky(Sinv)
+ Z = solve_triangular(R, beta*XY, check_finite=True, lower=True)
+ Mn = solve_triangular(R.T, Z, check_finite=True, lower=False)
+
+ # invert lower triangular matrix from cholesky decomposition
+ Ri = solve_triangular(R, np.eye(A.shape[0]), check_finite=False,
+ lower=True)
+ if full_covar:
+ Sn = np.dot(Ri.T, Ri)
+ return Mn, Sn, cholesky
+ else:
+ return Mn, Ri, cholesky
+ except LinAlgError:
+ cholesky = False
+ Sn = pinvh(Sinv)
+ Mn = beta*np.dot(Sinv, XY)
+ return Mn, Sn, cholesky
+
+ def _sparsity_quality(self, XX, XXd, XY, XYa, Aa, Ri, active, beta, cholesky):
+ '''
+ Calculates sparsity and quality parameters for each feature
+
+ Theoretical Note:
+ -----------------
+ Here we used Woodbury Identity for inverting covariance matrix
+ of target distribution
+ C = 1/beta + 1/alpha * X' * X
+ C^-1 = beta - beta^2 * X * Sn * X'
+ '''
+ bxy = beta*XY
+ bxx = beta*XXd
+ if cholesky:
+ # here Ri is inverse of lower triangular matrix obtained from
+ # cholesky decomp
+ xxr = np.dot(XX[:, active], Ri.T)
+ rxy = np.dot(Ri, XYa)
+ S = bxx - beta**2 * np.sum(xxr**2, axis=1)
+ Q = bxy - beta**2 * np.dot(xxr, rxy)
+ else:
+ # here Ri is covariance matrix
+ XXa = XX[:, active]
+ XS = np.dot(XXa, Ri)
+ S = bxx - beta**2 * np.sum(XS*XXa, 1)
+ Q = bxy - beta**2 * np.dot(XS, XYa)
+ # Use following:
+ # (EQ 1) q = A*Q/(A - S) ; s = A*S/(A-S)
+ # so if A = np.PINF q = Q, s = S
+ qi = np.copy(Q)
+ si = np.copy(S)
+ # If A is not np.PINF, then it should be 'active' feature => use (EQ 1)
+ Qa, Sa = Q[active], S[active]
+ qi[active] = Aa * Qa / (Aa - Sa)
+ si[active] = Aa * Sa / (Aa - Sa)
+
+ return [si, qi, S, Q]
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_laplace.py b/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_laplace.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fdcb5cf6e93c396d32eae2b0aad87a194a9cba4
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/reg_fast_laplace.py
@@ -0,0 +1,452 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+import numpy as np
+from sklearn.utils import as_float_array
+from sklearn.model_selection import KFold
+
+
+class RegressionFastLaplace():
+ '''
+ Sparse regression with Bayesian Compressive Sensing as described in Alg. 1
+ (Fast Laplace) of Ref.[1], which updated formulas from [2].
+
+ sigma2: noise precision (sigma^2)
+ nu fixed to 0
+
+ uqlab/lib/uq_regression/BCS/uq_bsc.m
+
+ Parameters
+ ----------
+ n_iter: int, optional (DEFAULT = 1000)
+ Maximum number of iterations
+
+ tol: float, optional (DEFAULT = 1e-7)
+ If absolute change in precision parameter for weights is below
+ threshold algorithm terminates.
+
+ fit_intercept : boolean, optional (DEFAULT = True)
+ whether to calculate the intercept for this model. If set
+ to false, no intercept will be used in calculations
+ (e.g. data is expected to be already centered).
+
+ copy_X : boolean, optional (DEFAULT = True)
+ If True, X will be copied; else, it may be overwritten.
+
+ verbose : boolean, optional (DEFAULT = FALSE)
+ Verbose mode when fitting the model
+
+ Attributes
+ ----------
+ coef_ : array, shape = (n_features)
+ Coefficients of the regression model (mean of posterior distribution)
+
+ alpha_ : float
+ estimated precision of the noise
+
+ active_ : array, dtype = np.bool, shape = (n_features)
+ True for non-zero coefficients, False otherwise
+
+ lambda_ : array, shape = (n_features)
+ estimated precisions of the coefficients
+
+ sigma_ : array, shape = (n_features, n_features)
+ estimated covariance matrix of the weights, computed only
+ for non-zero coefficients
+
+ References
+ ----------
+ [1] Babacan, S. D., Molina, R., & Katsaggelos, A. K. (2009). Bayesian
+ compressive sensing using Laplace priors. IEEE Transactions on image
+ processing, 19(1), 53-63.
+ [2] Fast marginal likelihood maximisation for sparse Bayesian models
+ (Tipping & Faul 2003).
+ (http://www.miketipping.com/papers/met-fastsbl.pdf)
+ '''
+
+ def __init__(self, n_iter=1000, n_Kfold=10, tol=1e-7, fit_intercept=False,
+ bias_term=True, copy_X=True, verbose=False):
+ self.n_iter = n_iter
+ self.n_Kfold = n_Kfold
+ self.tol = tol
+ self.fit_intercept = fit_intercept
+ self.bias_term = bias_term
+ self.copy_X = copy_X
+ self.verbose = verbose
+
+ def _center_data(self, X, y):
+ ''' Centers data'''
+ X = as_float_array(X, copy = self.copy_X)
+
+ # normalisation should be done in preprocessing!
+ X_std = np.ones(X.shape[1], dtype=X.dtype)
+ if self.fit_intercept:
+ X_mean = np.average(X, axis=0)
+ y_mean = np.average(y, axis=0)
+ X -= X_mean
+ y -= y_mean
+ else:
+ X_mean = np.zeros(X.shape[1], dtype=X.dtype)
+ y_mean = 0. if y.ndim == 1 else np.zeros(y.shape[1], dtype=X.dtype)
+ return X, y, X_mean, y_mean, X_std
+
+ def fit(self, X, y):
+
+ k_fold = KFold(n_splits=self.n_Kfold)
+
+ varY = np.var(y, ddof=1) if np.var(y, ddof=1) != 0 else 1.0
+ sigma2s = len(y)*varY*(10**np.linspace(-16, -1, self.n_Kfold))
+
+ errors = np.zeros((len(sigma2s), self.n_Kfold))
+ for s, sigma2 in enumerate(sigma2s):
+ for k, (train, test) in enumerate(k_fold.split(X, y)):
+ self.fit_(X[train], y[train], sigma2)
+ errors[s, k] = np.linalg.norm(
+ y[test] - self.predict(X[test])
+ )**2/len(test)
+
+ KfCVerror = np.sum(errors, axis=1)/self.n_Kfold/varY
+ i_minCV = np.argmin(KfCVerror)
+
+ self.kfoldCVerror = np.min(KfCVerror)
+
+ return self.fit_(X, y, sigma2s[i_minCV])
+
+ def fit_(self, X, y, sigma2):
+
+ N, P = X.shape
+ # n_samples, n_features = X.shape
+
+ X, y, X_mean, y_mean, X_std = self._center_data(X, y)
+ self._x_mean_ = X_mean
+ self._y_mean = y_mean
+ self._x_std = X_std
+
+ # check that variance is non zero !!!
+ if np.var(y) == 0:
+ self.var_y = True
+ else:
+ self.var_y = False
+ beta = 1./sigma2
+
+ # precompute X'*Y , X'*X for faster iterations & allocate memory for
+ # sparsity & quality vectors X=Psi
+ PsiTY = np.dot(X.T, y)
+ PsiTPsi = np.dot(X.T, X)
+ XXd = np.diag(PsiTPsi)
+
+ # initialize with constant regressor, or if that one does not exist,
+ # with the one that has the largest correlation with Y
+ ind_global_to_local = np.zeros(P, dtype=np.int32)
+
+ # identify constant regressors
+ constidx = np.where(~np.diff(X, axis=0).all(axis=0))[0]
+
+ if self.bias_term and constidx.size != 0:
+ ind_start = constidx[0]
+ ind_global_to_local[ind_start] = True
+ else:
+ # start from a single basis vector with largest projection on
+ # targets
+ proj = np.divide(np.square(PsiTY), XXd)
+ ind_start = np.argmax(proj)
+ ind_global_to_local[ind_start] = True
+
+ num_active = 1
+ active_indices = [ind_start]
+ deleted_indices = []
+ bcs_path = [ind_start]
+ gamma = np.zeros(P)
+ # for the initial value of gamma(ind_start), use the RVM formula
+ # gamma = (q^2 - s) / (s^2)
+ # and the fact that initially s = S = beta*Psi_i'*Psi_i and q = Q =
+ # beta*Psi_i'*Y
+ gamma[ind_start] = np.square(PsiTY[ind_start])
+ gamma[ind_start] -= sigma2 * PsiTPsi[ind_start, ind_start]
+ gamma[ind_start] /= np.square(PsiTPsi[ind_start, ind_start])
+
+ Sigma = 1. / (beta * PsiTPsi[ind_start, ind_start]
+ + 1./gamma[ind_start])
+
+ mu = Sigma * PsiTY[ind_start] * beta
+ tmp1 = beta * PsiTPsi[ind_start]
+ S = beta * np.diag(PsiTPsi).T - Sigma * np.square(tmp1)
+ Q = beta * PsiTY.T - mu*(tmp1)
+
+ tmp2 = np.ones(P) # alternative computation for the initial s,q
+ q0tilde = PsiTY[ind_start]
+ s0tilde = PsiTPsi[ind_start, ind_start]
+ tmp2[ind_start] = s0tilde / (q0tilde**2) / beta
+ s = np.divide(S, tmp2)
+ q = np.divide(Q, tmp2)
+ Lambda = 2*(num_active - 1) / np.sum(gamma)
+
+ Delta_L_max = []
+ for i in range(self.n_iter):
+ # Handle variance zero
+ if self.var_y:
+ mu = np.mean(y)
+ break
+
+ if self.verbose:
+ print(' lambda = {0:.6e}\n'.format(Lambda))
+
+ # Calculate the potential updated value of each gamma[i]
+ if Lambda == 0.0: # RVM
+ gamma_potential = np.multiply((
+ (q**2 - s) > Lambda),
+ np.divide(q**2 - s, s**2)
+ )
+ else:
+ a = Lambda * s**2
+ b = s**2 + 2*Lambda*s
+ c = Lambda + s - q**2
+ gamma_potential = np.multiply(
+ (c < 0), np.divide(
+ -b + np.sqrt(b**2 - 4*np.multiply(a, c)), 2*a)
+ )
+
+ l_gamma = - np.log(np.absolute(1 + np.multiply(gamma, s)))
+ l_gamma += np.divide(np.multiply(q**2, gamma),
+ (1 + np.multiply(gamma, s)))
+ l_gamma -= Lambda*gamma # omitted the factor 1/2
+
+ # Contribution of each updated gamma(i) to L(gamma)
+ l_gamma_potential = - np.log(
+ np.absolute(1 + np.multiply(gamma_potential, s))
+ )
+ l_gamma_potential += np.divide(
+ np.multiply(q**2, gamma_potential),
+ (1 + np.multiply(gamma_potential, s))
+ )
+ # omitted the factor 1/2
+ l_gamma_potential -= Lambda*gamma_potential
+
+ # Check how L(gamma) would change if we replaced gamma(i) by the
+ # updated gamma_potential(i), for each i separately
+ Delta_L_potential = l_gamma_potential - l_gamma
+
+ # deleted indices should not be chosen again
+ if len(deleted_indices) != 0:
+ values = -np.inf * np.ones(len(deleted_indices))
+ Delta_L_potential[deleted_indices] = values
+
+ Delta_L_max.append(np.nanmax(Delta_L_potential))
+ ind_L_max = np.nanargmax(Delta_L_potential)
+
+ # in case there is only 1 regressor in the model and it would now
+ # be deleted
+ if len(active_indices) == 1 and ind_L_max == active_indices[0] \
+ and gamma_potential[ind_L_max] == 0.0:
+ Delta_L_potential[ind_L_max] = -np.inf
+ Delta_L_max[i] = np.max(Delta_L_potential)
+ ind_L_max = np.argmax(Delta_L_potential)
+
+ # If L did not change significantly anymore, break
+ if Delta_L_max[i] <= 0.0 or\
+ (i > 0 and all(np.absolute(Delta_L_max[i-1:])
+ < sum(Delta_L_max)*self.tol)) or \
+ (i > 0 and all(np.diff(bcs_path)[i-1:] == 0.0)):
+ if self.verbose:
+ print('Increase in L: {0:.6e} (eta = {1:.3e})\
+ -- break\n'.format(Delta_L_max[i], self.tol))
+ break
+
+ # Print information
+ if self.verbose:
+ print(' Delta L = {0:.6e} \n'.format(Delta_L_max[i]))
+
+ what_changed = int(gamma[ind_L_max] == 0.0)
+ what_changed -= int(gamma_potential[ind_L_max] == 0.0)
+
+ # Print information
+ if self.verbose:
+ if what_changed < 0:
+ print(f'{i+1} - Remove regressor #{ind_L_max+1}..\n')
+ elif what_changed == 0:
+ print(f'{i+1} - Recompute regressor #{ind_L_max+1}..\n')
+ else:
+ print(f'{i+1} - Add regressor #{ind_L_max+1}..\n')
+
+ # --- Update all quantities ----
+ if what_changed == 1:
+ # adding a regressor
+
+ # update gamma
+ gamma[ind_L_max] = gamma_potential[ind_L_max]
+
+ Sigma_ii = 1.0 / (1.0/gamma[ind_L_max] + S[ind_L_max])
+ try:
+ x_i = np.matmul(
+ Sigma, PsiTPsi[active_indices, ind_L_max].reshape(-1, 1)
+ )
+ except ValueError:
+ x_i = Sigma * PsiTPsi[active_indices, ind_L_max]
+ tmp_1 = - (beta * Sigma_ii) * x_i
+ Sigma = np.vstack(
+ (np.hstack(((beta**2 * Sigma_ii) * np.dot(x_i, x_i.T)
+ + Sigma, tmp_1)), np.append(tmp_1.T, Sigma_ii))
+ )
+ mu_i = Sigma_ii * Q[ind_L_max]
+ mu = np.vstack((mu - (beta * mu_i) * x_i, mu_i))
+
+ tmp2_1 = PsiTPsi[:, ind_L_max] - beta * np.squeeze(
+ np.matmul(PsiTPsi[:, active_indices], x_i)
+ )
+ if i == 0:
+ tmp2_1[0] /= 2
+ tmp2 = beta * tmp2_1.T
+ S = S - Sigma_ii * np.square(tmp2)
+ Q = Q - mu_i * tmp2
+
+ num_active += 1
+ ind_global_to_local[ind_L_max] = num_active
+ active_indices.append(ind_L_max)
+ bcs_path.append(ind_L_max)
+
+ elif what_changed == 0:
+ # recomputation
+ # zero if regressor has not been chosen yet
+ if not ind_global_to_local[ind_L_max]:
+ raise Exception('Cannot recompute index{0} -- not yet\
+ part of the model!'.format(ind_L_max))
+ Sigma = np.atleast_2d(Sigma)
+ mu = np.atleast_2d(mu)
+ gamma_i_new = gamma_potential[ind_L_max]
+ gamma_i_old = gamma[ind_L_max]
+ # update gamma
+ gamma[ind_L_max] = gamma_potential[ind_L_max]
+
+ # index of regressor in Sigma
+ local_ind = ind_global_to_local[ind_L_max]-1
+
+ kappa_i = (1.0/gamma_i_new - 1.0/gamma_i_old)
+ kappa_i = 1.0 / kappa_i
+ kappa_i += Sigma[local_ind, local_ind]
+ kappa_i = 1 / kappa_i
+ Sigma_i_col = Sigma[:, local_ind]
+
+ Sigma = Sigma - kappa_i * (Sigma_i_col * Sigma_i_col.T)
+ mu_i = mu[local_ind]
+ mu = mu - (kappa_i * mu_i) * Sigma_i_col[:, None]
+
+ tmp1 = beta * np.dot(
+ Sigma_i_col.reshape(1, -1), PsiTPsi[active_indices])[0]
+ S = S + kappa_i * np.square(tmp1)
+ Q = Q + (kappa_i * mu_i) * tmp1
+
+ # no change in active_indices or ind_global_to_local
+ bcs_path.append(ind_L_max + 0.1)
+
+ elif what_changed == -1:
+ gamma[ind_L_max] = 0
+
+ # index of regressor in Sigma
+ local_ind = ind_global_to_local[ind_L_max]-1
+
+ Sigma_ii_inv = 1. / Sigma[local_ind, local_ind]
+ Sigma_i_col = Sigma[:, local_ind]
+
+ Sigma = Sigma - Sigma_ii_inv * (Sigma_i_col * Sigma_i_col.T)
+
+ Sigma = np.delete(
+ np.delete(Sigma, local_ind, axis=0), local_ind, axis=1)
+
+ mu = mu - (mu[local_ind] * Sigma_ii_inv) * Sigma_i_col[:, None]
+ mu = np.delete(mu, local_ind, axis=0)
+
+ tmp1 = beta * np.dot(Sigma_i_col, PsiTPsi[active_indices])
+ S = S + Sigma_ii_inv * np.square(tmp1)
+ Q = Q + (mu_i * Sigma_ii_inv) * tmp1
+
+ num_active -= 1
+ ind_global_to_local[ind_L_max] = 0.0
+ v = ind_global_to_local[ind_global_to_local > local_ind] - 1
+ ind_global_to_local[ind_global_to_local > local_ind] = v
+ del active_indices[local_ind]
+ deleted_indices.append(ind_L_max)
+ # and therefore ineligible
+ bcs_path.append(-ind_L_max)
+
+ # same for all three cases
+ tmp3 = 1 - np.multiply(gamma, S)
+ s = np.divide(S, tmp3)
+ q = np.divide(Q, tmp3)
+
+ # Update lambda
+ Lambda = 2*(num_active - 1) / np.sum(gamma)
+
+ # Prepare the result object
+ self.coef_ = np.zeros(P)
+ self.coef_[active_indices] = np.squeeze(mu)
+ self.sigma_ = Sigma
+ self.active_ = active_indices
+ self.gamma = gamma
+ self.Lambda = Lambda
+ self.beta = beta
+ self.bcs_path = bcs_path
+
+ # set intercept_
+ if self.fit_intercept:
+ self.coef_ = self.coef_ / X_std
+ self.intercept_ = y_mean - np.dot(X_mean, self.coef_.T)
+ else:
+ self.intercept_ = 0.
+
+ return self
+
+ def predict(self, X, return_std=False):
+ '''
+ Computes predictive distribution for test set.
+ Predictive distribution for each data point is one dimensional
+ Gaussian and therefore is characterised by mean and variance based on
+ Ref.[1] Section 3.3.2.
+
+ Parameters
+ -----------
+ X: {array-like, sparse} (n_samples_test, n_features)
+ Test data, matrix of explanatory variables
+
+ Returns
+ -------
+ : list of length two [y_hat, var_hat]
+
+ y_hat: numpy array of size (n_samples_test,)
+ Estimated values of targets on test set (i.e. mean of
+ predictive distribution)
+
+ var_hat: numpy array of size (n_samples_test,)
+ Variance of predictive distribution
+
+ References
+ ----------
+ [1] Bishop, C. M. (2006). Pattern recognition and machine learning.
+ springer.
+ '''
+ y_hat = np.dot(X, self.coef_) + self.intercept_
+
+ if return_std:
+ # Handle the zero variance case
+ if self.var_y:
+ return y_hat, np.zeros_like(y_hat)
+
+ var_hat = 1./self.beta
+ var_hat += np.sum(X.dot(self.sigma_) * X, axis=1)
+ std_hat = np.sqrt(var_hat)
+ return y_hat, std_hat
+ else:
+ return y_hat
+
+# l2norm = 0.0
+# for idx in range(10):
+# sigma2 = np.genfromtxt('./test/sigma2_{0}.csv'.format(idx+1), delimiter=',')
+# Psi_train = np.genfromtxt('./test/Psi_train_{0}.csv'.format(idx+1), delimiter=',')
+# Y_train = np.genfromtxt('./test/Y_train_{0}.csv'.format(idx+1))
+# Psi_test = np.genfromtxt('./test/Psi_test_{0}.csv'.format(idx+1), delimiter=',')
+# Y_test = np.genfromtxt('./test/Y_test_{0}.csv'.format(idx+1))
+
+# clf = RegressionFastLaplace(verbose=True)
+# clf.fit_(Psi_train, Y_train, sigma2)
+# coeffs_fold = np.genfromtxt('./test/coeffs_fold_{0}.csv'.format(idx+1))
+# print("coeffs error: {0:.4g}".format(np.linalg.norm(clf.coef_ - coeffs_fold)))
+# l2norm += np.linalg.norm(Y_test - clf.predict(Psi_test))**2/len(Y_test)
+# print("l2norm error: {0:.4g}".format(l2norm))
diff --git a/examples/model-comparison/bayesvalidrox/surrogate_models/surrogate_models.py b/examples/model-comparison/bayesvalidrox/surrogate_models/surrogate_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..ca902f26bef0c45e8befb72ff67313ef09a77603
--- /dev/null
+++ b/examples/model-comparison/bayesvalidrox/surrogate_models/surrogate_models.py
@@ -0,0 +1,1576 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Implementation of metamodel as either PC, aPC or GPE
+"""
+
+import warnings
+import numpy as np
+import math
+import h5py
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import MinMaxScaler
+import scipy as sp
+from scipy.optimize import minimize, NonlinearConstraint, LinearConstraint
+from tqdm import tqdm
+from sklearn.decomposition import PCA as sklearnPCA
+import sklearn.linear_model as lm
+from sklearn.gaussian_process import GaussianProcessRegressor
+import sklearn.gaussian_process.kernels as kernels
+import os
+from joblib import Parallel, delayed
+import copy
+
+from .input_space import InputSpace
+from .glexindex import glexindex
+from .eval_rec_rule import eval_univ_basis
+from .reg_fast_ard import RegressionFastARD
+from .reg_fast_laplace import RegressionFastLaplace
+from .orthogonal_matching_pursuit import OrthogonalMatchingPursuit
+from .bayes_linear import VBLinearRegression, EBLinearRegression
+from .apoly_construction import apoly_construction
+warnings.filterwarnings("ignore")
+# Load the mplstyle
+plt.style.use(os.path.join(os.path.split(__file__)[0],
+ '../', 'bayesvalidrox.mplstyle'))
+
+
+class MetaModel():
+ """
+ Meta (surrogate) model
+
+ This class trains a surrogate model. It accepts an input object (input_obj)
+ containing the specification of the distributions for uncertain parameters
+ and a model object with instructions on how to run the computational model.
+
+ Attributes
+ ----------
+ input_obj : obj
+ Input object with the information on the model input parameters.
+ meta_model_type : str
+ Surrogate model types. Three surrogate model types are supported:
+ polynomial chaos expansion (`PCE`), arbitrary PCE (`aPCE`) and
+ Gaussian process regression (`GPE`). Default is PCE.
+ pce_reg_method : str
+ PCE regression method to compute the coefficients. The following
+ regression methods are available:
+
+ 1. OLS: Ordinary Least Square method
+ 2. BRR: Bayesian Ridge Regression
+ 3. LARS: Least angle regression
+ 4. ARD: Bayesian ARD Regression
+ 5. FastARD: Fast Bayesian ARD Regression
+ 6. VBL: Variational Bayesian Learning
+ 7. EBL: Emperical Bayesian Learning
+ Default is `OLS`.
+ bootstrap_method : str
+ Bootstraping method. Options are `'normal'` and `'fast'`. The default
+ is `'fast'`. It means that in each iteration except the first one, only
+ the coefficent are recalculated with the ordinary least square method.
+ n_bootstrap_itrs : int
+ Number of iterations for the bootstrap sampling. The default is `1`.
+ pce_deg : int or list of int
+ Polynomial degree(s). If a list is given, an adaptive algorithm is used
+ to find the best degree with the lowest Leave-One-Out cross-validation
+ (LOO) error (or the highest score=1-LOO). Default is `1`.
+ pce_q_norm : float
+ Hyperbolic (or q-norm) truncation for multi-indices of multivariate
+ polynomials. Default is `1.0`.
+ dim_red_method : str
+ Dimensionality reduction method for the output space. The available
+ method is based on principal component analysis (PCA). The Default is
+ `'no'`. There are two ways to select number of components: use
+ percentage of the explainable variance threshold (between 0 and 100)
+ (Option A) or direct prescription of components' number (Option B):
+
+ >>> MetaModelOpts.dim_red_method = 'PCA'
+ >>> MetaModelOpts.var_pca_threshold = 99.999 # Option A
+ >>> MetaModelOpts.n_pca_components = 12 # Option B
+ apply_constraints : bool
+ If set to true constraints will be applied during training.
+ In this case the training uses OLS. In this version the constraints
+ need to be set explicitly in this class.
+
+ verbose : bool
+ Prints summary of the regression results. Default is `False`.
+
+ Note
+ -------
+ To define the sampling methods and the training set, an experimental design
+ instance shall be defined. This can be done by:
+
+ >>> MetaModelOpts.add_InputSpace()
+
+ Two experimental design schemes are supported: one-shot (`normal`) and
+ adaptive sequential (`sequential`) designs.
+ For experimental design refer to `InputSpace`.
+
+ """
+
+ def __init__(self, input_obj, meta_model_type='PCE',
+ pce_reg_method='OLS', bootstrap_method='fast',
+ n_bootstrap_itrs=1, pce_deg=1, pce_q_norm=1.0,
+ dim_red_method='no', apply_constraints = False,
+ verbose=False):
+
+ self.input_obj = input_obj
+ self.meta_model_type = meta_model_type
+ self.pce_reg_method = pce_reg_method
+ self.bootstrap_method = bootstrap_method
+ self.n_bootstrap_itrs = n_bootstrap_itrs
+ self.pce_deg = pce_deg
+ self.pce_q_norm = pce_q_norm
+ self.dim_red_method = dim_red_method
+ self.apply_constraints = apply_constraints
+ self.verbose = verbose
+
+ def build_metamodel(self, n_init_samples = None) -> None:
+ """
+ Builds the parts for the metamodel (polynomes,...) that are neede before fitting.
+
+ Returns
+ -------
+ None
+ DESCRIPTION.
+
+ """
+
+ # Generate general warnings
+ if self.apply_constraints or self.pce_reg_method.lower() == 'ols':
+ print('There are no estimations of surrogate uncertainty available'
+ ' for the chosen regression options. This might lead to issues'
+ ' in later steps.')
+
+ # Add InputSpace to MetaModel if it does not have any
+ if not hasattr(self, 'InputSpace'):
+ self.InputSpace = InputSpace(self.input_obj)
+ self.InputSpace.n_init_samples = n_init_samples
+ self.InputSpace.init_param_space(np.max(self.pce_deg))
+
+ self.ndim = self.InputSpace.ndim
+
+ if not hasattr(self, 'CollocationPoints'):
+ raise AttributeError('Please provide samples to the metamodel before building it.')
+
+ # Transform input samples
+ # TODO: this is probably not yet correct! Make 'method' variable
+ self.CollocationPoints = self.InputSpace.transform(self.CollocationPoints, method='user')
+
+
+ self.n_params = len(self.input_obj.Marginals)
+
+ # Generate polynomials
+ if self.meta_model_type.lower() != 'gpe':
+ self.generate_polynomials(np.max(self.pce_deg))
+
+ # Initialize the nested dictionaries
+ if self.meta_model_type.lower() == 'gpe':
+ self.gp_poly = self.auto_vivification()
+ self.x_scaler = self.auto_vivification()
+ self.LCerror = self.auto_vivification()
+ else:
+ self.deg_dict = self.auto_vivification()
+ self.q_norm_dict = self.auto_vivification()
+ self.coeffs_dict = self.auto_vivification()
+ self.basis_dict = self.auto_vivification()
+ self.score_dict = self.auto_vivification()
+ self.clf_poly = self.auto_vivification()
+ self.LCerror = self.auto_vivification()
+ if self.dim_red_method.lower() == 'pca':
+ self.pca = self.auto_vivification()
+
+ # Define an array containing the degrees
+ self.CollocationPoints = np.array(self.CollocationPoints)
+ self.n_samples, ndim = self.CollocationPoints.shape
+ if self.ndim != ndim:
+ raise AttributeError('The given samples do not match the given number of priors. The samples should be a 2D array of size (#samples, #priors)')
+
+ self.deg_array = self.__select_degree(ndim, self.n_samples)
+
+ # Generate all basis indices
+ self.allBasisIndices = self.auto_vivification()
+ for deg in self.deg_array:
+ keys = self.allBasisIndices.keys()
+ if deg not in np.fromiter(keys, dtype=float):
+ # Generate the polynomial basis indices
+ for qidx, q in enumerate(self.pce_q_norm):
+ basis_indices = glexindex(start=0, stop=deg+1,
+ dimensions=self.n_params,
+ cross_truncation=q,
+ reverse=False, graded=True)
+ self.allBasisIndices[str(deg)][str(q)] = basis_indices
+
+
+
+ def fit(self, X, y, parallel = True, verbose = False):
+ """
+ Fits the surrogate to the given data (samples X, outputs y).
+ Note here that the samples X should be the transformed samples provided
+ by the experimental design if the transformation is used there.
+
+ Parameters
+ ----------
+ X : 2D list or np.array of shape (#samples, #dim)
+ The parameter value combinations that the model was evaluated at.
+ y : dict of 2D lists or arrays of shape (#samples, #timesteps)
+ The respective model evaluations.
+
+ Returns
+ -------
+ None.
+
+ """
+ X = np.array(X)
+ for key in y.keys():
+ y_val = np.array(y[key])
+ if y_val.ndim !=2:
+ raise ValueError('The given outputs y should be 2D')
+ y[key] = np.array(y[key])
+
+ # Output names are the same as the keys in y
+ self.out_names = list(y.keys())
+
+ # Build the MetaModel on the static samples
+ self.CollocationPoints = X
+
+ # TODO: other option: rebuild every time
+ if not hasattr(self, 'deg_array'):
+ self.build_metamodel(n_init_samples = X.shape[1])
+
+ # Evaluate the univariate polynomials on InputSpace
+ if self.meta_model_type.lower() != 'gpe':
+ self.univ_p_val = self.univ_basis_vals(self.CollocationPoints)
+
+ # --- Loop through data points and fit the surrogate ---
+ if verbose:
+ print(f"\n>>>> Training the {self.meta_model_type} metamodel "
+ "started. <<<<<<\n")
+
+ # --- Bootstrap sampling ---
+ # Correct number of bootstrap if PCA transformation is required.
+ if self.dim_red_method.lower() == 'pca' and self.n_bootstrap_itrs == 1:
+ self.n_bootstrap_itrs = 100
+
+ # Check if fast version (update coeffs with OLS) is selected.
+ if self.bootstrap_method.lower() == 'fast':
+ fast_bootstrap = True
+ first_out = {}
+ n_comp_dict = {}
+ else:
+ fast_bootstrap = False
+
+ # Prepare tqdm iteration maessage
+ if verbose and self.n_bootstrap_itrs > 1:
+ enum_obj = tqdm(range(self.n_bootstrap_itrs),
+ total=self.n_bootstrap_itrs,
+ desc="Bootstrapping the metamodel",
+ ascii=True)
+ else:
+ enum_obj = range(self.n_bootstrap_itrs)
+
+ # Loop over the bootstrap iterations
+ for b_i in enum_obj:
+ if b_i > 0:
+ b_indices = np.random.randint(self.n_samples, size=self.n_samples)
+ else:
+ b_indices = np.arange(len(X))
+
+ X_train_b = X[b_indices]
+
+ if verbose and self.n_bootstrap_itrs == 1:
+ items = tqdm(y.items(), desc="Fitting regression")
+ else:
+ items = y.items()
+
+ # For loop over the components/outputs
+ for key, Output in items:
+
+ # Dimensionality reduction with PCA, if specified
+ if self.dim_red_method.lower() == 'pca':
+
+ # Use the stored n_comp for fast bootsrtrapping
+ if fast_bootstrap and b_i > 0:
+ self.n_pca_components = n_comp_dict[key]
+
+ # Start transformation
+ pca, target, n_comp = self.pca_transformation(
+ Output[b_indices], verbose=False
+ )
+ self.pca[f'b_{b_i+1}'][key] = pca
+ # Store the number of components for fast bootsrtrapping
+ if fast_bootstrap and b_i == 0:
+ n_comp_dict[key] = n_comp
+ else:
+ target = Output[b_indices]
+
+ # Parallel fit regression
+ if self.meta_model_type.lower() == 'gpe':
+ # Prepare the input matrix
+ scaler = MinMaxScaler()
+ X_S = scaler.fit_transform(X_train_b)
+
+ self.x_scaler[f'b_{b_i+1}'][key] = scaler
+ if parallel:
+ out = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.gaussian_process_emulator)(
+ X_S, target[:, idx]) for idx in
+ range(target.shape[1]))
+ else:
+ results = map(self.gaussian_process_emulator,
+ [X_train_b]*target.shape[1],
+ [target[:, idx] for idx in
+ range(target.shape[1])]
+ )
+ out = list(results)
+
+ for idx in range(target.shape[1]):
+ self.gp_poly[f'b_{b_i+1}'][key][f"y_{idx+1}"] = out[idx]
+
+ else:
+ self.univ_p_val = self.univ_p_val[b_indices]
+ if parallel and (not fast_bootstrap or b_i == 0):
+ out = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.adaptive_regression)(X_train_b,
+ target[:, idx],
+ idx)
+ for idx in range(target.shape[1]))
+ elif not parallel and (not fast_bootstrap or b_i == 0):
+ results = map(self.adaptive_regression,
+ [X_train_b]*target.shape[1],
+ [target[:, idx] for idx in
+ range(target.shape[1])],
+ range(target.shape[1]))
+ out = list(results)
+
+ # Store the first out dictionary
+ if fast_bootstrap and b_i == 0:
+ first_out[key] = copy.deepcopy(out)
+
+ if b_i > 0 and fast_bootstrap:
+
+ # fast bootstrap
+ out = self.update_pce_coeffs(
+ X_train_b, target, first_out[key])
+
+ for i in range(target.shape[1]):
+ # Create a dict to pass the variables
+ self.deg_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['degree']
+ self.q_norm_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['qnorm']
+ self.coeffs_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['coeffs']
+ self.basis_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['multi_indices']
+ self.score_dict[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['LOOCVScore']
+ self.clf_poly[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['clf_poly']
+ #self.LCerror[f'b_{b_i+1}'][key][f"y_{i+1}"] = out[i]['LCerror']
+
+ if verbose:
+ print(f"\n>>>> Training the {self.meta_model_type} metamodel"
+ " sucessfully completed. <<<<<<\n")
+
+ # -------------------------------------------------------------------------
+ def update_pce_coeffs(self, X, y, out_dict = None):
+ """
+ Updates the PCE coefficents using only the ordinary least square method
+ for the fast version of the bootstrapping.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_params)
+ Training set.
+ y : array of shape (n_samples, n_outs)
+ The (transformed) model responses.
+ out_dict : dict
+ The training output dictionary of the first iteration, i.e.
+ the surrogate model for the original experimental design.
+
+ Returns
+ -------
+ final_out_dict : dict
+ The updated training output dictionary.
+
+ """
+ # Make a copy
+ final_out_dict = copy.deepcopy(out_dict)
+
+ # Loop over the points
+ for i in range(y.shape[1]):
+
+
+ # Extract nonzero basis indices
+ nnz_idx = np.nonzero(out_dict[i]['coeffs'])[0]
+ if len(nnz_idx) != 0:
+ basis_indices = out_dict[i]['multi_indices']
+
+ # Evaluate the multivariate polynomials on CollocationPoints
+ psi = self.create_psi(basis_indices, self.univ_p_val)
+
+ # Calulate the cofficients of surrogate model
+ updated_out = self.regression(
+ psi, y[:, i], basis_indices, reg_method='OLS',
+ sparsity=False
+ )
+
+ # Update coeffs in out_dict
+ final_out_dict[i]['coeffs'][nnz_idx] = updated_out['coeffs']
+
+ return final_out_dict
+
+ # -------------------------------------------------------------------------
+ def add_InputSpace(self):
+ """
+ Instanciates experimental design object.
+
+ Returns
+ -------
+ None.
+
+ """
+ self.InputSpace = InputSpace(self.input_obj,
+ meta_Model_type=self.meta_model_type)
+
+ # -------------------------------------------------------------------------
+ def univ_basis_vals(self, samples, n_max=None):
+ """
+ Evaluates univariate regressors along input directions.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params)
+ Samples.
+ n_max : int, optional
+ Maximum polynomial degree. The default is `None`.
+
+ Returns
+ -------
+ univ_basis: array of shape (n_samples, n_params, n_max+1)
+ All univariate regressors up to n_max.
+ """
+ # Extract information
+ poly_types = self.InputSpace.poly_types
+ if samples.ndim != 2:
+ samples = samples.reshape(1, len(samples))
+ n_max = np.max(self.pce_deg) if n_max is None else n_max
+
+ # Extract poly coeffs
+ if self.InputSpace.input_data_given or self.InputSpace.apce:
+ apolycoeffs = self.polycoeffs
+ else:
+ apolycoeffs = None
+
+ # Evaluate univariate basis
+ univ_basis = eval_univ_basis(samples, n_max, poly_types, apolycoeffs)
+
+ return univ_basis
+
+ # -------------------------------------------------------------------------
+ def create_psi(self, basis_indices, univ_p_val):
+ """
+ This function assemble the design matrix Psi from the given basis index
+ set INDICES and the univariate polynomial evaluations univ_p_val.
+
+ Parameters
+ ----------
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ univ_p_val : array of (n_samples, n_params, n_max+1)
+ All univariate regressors up to `n_max`.
+
+ Raises
+ ------
+ ValueError
+ n_terms in arguments do not match.
+
+ Returns
+ -------
+ psi : array of shape (n_samples, n_terms)
+ Multivariate regressors.
+
+ """
+ # Check if BasisIndices is a sparse matrix
+ sparsity = sp.sparse.issparse(basis_indices)
+ if sparsity:
+ basis_indices = basis_indices.toarray()
+
+ # Initialization and consistency checks
+ # number of input variables
+ n_params = univ_p_val.shape[1]
+
+ # Size of the experimental design
+ n_samples = univ_p_val.shape[0]
+
+ # number of basis terms
+ n_terms = basis_indices.shape[0]
+
+ # check that the variables have consistent sizes
+ if n_params != basis_indices.shape[1]:
+ raise ValueError(
+ f"The shapes of basis_indices ({basis_indices.shape[1]}) and "
+ f"univ_p_val ({n_params}) don't match!!"
+ )
+
+ # Preallocate the Psi matrix for performance
+ psi = np.ones((n_samples, n_terms))
+ # Assemble the Psi matrix
+ for m in range(basis_indices.shape[1]):
+ aa = np.where(basis_indices[:, m] > 0)[0]
+ try:
+ basisIdx = basis_indices[aa, m]
+ bb = univ_p_val[:, m, basisIdx].reshape(psi[:, aa].shape)
+ psi[:, aa] = np.multiply(psi[:, aa], bb)
+ except ValueError as err:
+ raise err
+ return psi
+
+ # -------------------------------------------------------------------------
+ def regression(self, X, y, basis_indices, reg_method=None, sparsity=True):
+ """
+ Fit regression using the regression method provided.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_features)
+ Training vector, where n_samples is the number of samples and
+ n_features is the number of features.
+ y : array of shape (n_samples,)
+ Target values.
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ reg_method : str, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ return_out_dict : Dict
+ Fitted estimator, spareMulti-Index, sparseX and coefficients.
+
+ """
+ if reg_method is None:
+ reg_method = self.pce_reg_method
+
+ bias_term = self.dim_red_method.lower() != 'pca'
+
+ compute_score = True if self.verbose else False
+
+ # inverse of the observed variance of the data
+ if np.var(y) != 0:
+ Lambda = 1 / np.var(y)
+ else:
+ Lambda = 1e-6
+
+ # Bayes sparse adaptive aPCE
+ if reg_method.lower() == 'ols':
+ clf_poly = lm.LinearRegression(fit_intercept=False)
+ elif reg_method.lower() == 'brr':
+ clf_poly = lm.BayesianRidge(n_iter=1000, tol=1e-7,
+ fit_intercept=False,
+ #normalize=True,
+ compute_score=compute_score,
+ alpha_1=1e-04, alpha_2=1e-04,
+ lambda_1=Lambda, lambda_2=Lambda)
+ clf_poly.converged = True
+
+ elif reg_method.lower() == 'ard':
+ if X.shape[0]<2:
+ raise ValueError('Regression with ARD can only be performed for more than 2 samples')
+ clf_poly = lm.ARDRegression(fit_intercept=False,
+ #normalize=True,
+ compute_score=compute_score,
+ n_iter=1000, tol=0.0001,
+ alpha_1=1e-3, alpha_2=1e-3,
+ lambda_1=Lambda, lambda_2=Lambda)
+
+ elif reg_method.lower() == 'fastard':
+ clf_poly = RegressionFastARD(fit_intercept=False,
+ normalize=True,
+ compute_score=compute_score,
+ n_iter=300, tol=1e-10)
+
+ elif reg_method.lower() == 'bcs':
+ if X.shape[0]<10:
+ raise ValueError('Regression with BCS can only be performed for more than 10 samples')
+ clf_poly = RegressionFastLaplace(fit_intercept=False,
+ bias_term=bias_term,
+ n_iter=1000, tol=1e-7)
+
+ elif reg_method.lower() == 'lars':
+ if X.shape[0]<10:
+ raise ValueError('Regression with LARS can only be performed for more than 5 samples')
+ clf_poly = lm.LassoLarsCV(fit_intercept=False)
+
+ elif reg_method.lower() == 'sgdr':
+ clf_poly = lm.SGDRegressor(fit_intercept=False,
+ max_iter=5000, tol=1e-7)
+
+ elif reg_method.lower() == 'omp':
+ clf_poly = OrthogonalMatchingPursuit(fit_intercept=False)
+
+ elif reg_method.lower() == 'vbl':
+ clf_poly = VBLinearRegression(fit_intercept=False)
+
+ elif reg_method.lower() == 'ebl':
+ clf_poly = EBLinearRegression(optimizer='em')
+
+
+ # Training with constraints automatically uses L2
+ if self.apply_constraints:
+ # TODO: set the constraints here
+ # Define the nonlin. constraint
+ nlc = NonlinearConstraint(lambda x: np.matmul(X,x),-1,1.1)
+ self.nlc = nlc
+
+ fun = lambda x: (np.linalg.norm(np.matmul(X, x)-y, ord = 2))**2
+ if self.init_type =='zeros':
+ res = minimize(fun, np.zeros(X.shape[1]), method = 'trust-constr', constraints = self.nlc)
+ if self.init_type == 'nonpi':
+ clf_poly.fit(X, y)
+ coeff = clf_poly.coef_
+ res = minimize(fun, coeff, method = 'trust-constr', constraints = self.nlc)
+
+ coeff = np.array(res.x)
+ clf_poly.coef_ = coeff
+ clf_poly.X = X
+ clf_poly.y = y
+ clf_poly.intercept_ = 0
+
+ # Training without constraints uses chosen regression method
+ else:
+ clf_poly.fit(X, y)
+
+ # Select the nonzero entries of coefficients
+ if sparsity:
+ nnz_idx = np.nonzero(clf_poly.coef_)[0]
+ else:
+ nnz_idx = np.arange(clf_poly.coef_.shape[0])
+
+ # This is for the case where all outputs are zero, thereby
+ # all coefficients are zero
+ if (y == 0).all():
+ nnz_idx = np.insert(np.nonzero(clf_poly.coef_)[0], 0, 0)
+
+ sparse_basis_indices = basis_indices[nnz_idx]
+ sparse_X = X[:, nnz_idx]
+ coeffs = clf_poly.coef_[nnz_idx]
+ clf_poly.coef_ = coeffs
+
+ # Create a dict to pass the outputs
+ return_out_dict = dict()
+ return_out_dict['clf_poly'] = clf_poly
+ return_out_dict['spareMulti-Index'] = sparse_basis_indices
+ return_out_dict['sparePsi'] = sparse_X
+ return_out_dict['coeffs'] = coeffs
+ return return_out_dict
+
+ # -------------------------------------------------------------------------
+ def create_psi(self, basis_indices, univ_p_val):
+ """
+ This function assemble the design matrix Psi from the given basis index
+ set INDICES and the univariate polynomial evaluations univ_p_val.
+
+ Parameters
+ ----------
+ basis_indices : array of shape (n_terms, n_params)
+ Multi-indices of multivariate polynomials.
+ univ_p_val : array of (n_samples, n_params, n_max+1)
+ All univariate regressors up to `n_max`.
+
+ Raises
+ ------
+ ValueError
+ n_terms in arguments do not match.
+
+ Returns
+ -------
+ psi : array of shape (n_samples, n_terms)
+ Multivariate regressors.
+
+ """
+ # Check if BasisIndices is a sparse matrix
+ sparsity = sp.sparse.issparse(basis_indices)
+ if sparsity:
+ basis_indices = basis_indices.toarray()
+
+ # Initialization and consistency checks
+ # number of input variables
+ n_params = univ_p_val.shape[1]
+
+ # Size of the experimental design
+ n_samples = univ_p_val.shape[0]
+
+ # number of basis terms
+ n_terms = basis_indices.shape[0]
+
+ # check that the variables have consistent sizes
+ if n_params != basis_indices.shape[1]:
+ raise ValueError(
+ f"The shapes of basis_indices ({basis_indices.shape[1]}) and "
+ f"univ_p_val ({n_params}) don't match!!"
+ )
+
+ # Preallocate the Psi matrix for performance
+ psi = np.ones((n_samples, n_terms))
+ # Assemble the Psi matrix
+ for m in range(basis_indices.shape[1]):
+ aa = np.where(basis_indices[:, m] > 0)[0]
+ try:
+ basisIdx = basis_indices[aa, m]
+ bb = univ_p_val[:, m, basisIdx].reshape(psi[:, aa].shape)
+ psi[:, aa] = np.multiply(psi[:, aa], bb)
+ except ValueError as err:
+ raise err
+ return psi
+
+ # --------------------------------------------------------------------------------------------------------
+ def adaptive_regression(self, ED_X, ED_Y, varIdx, verbose=False):
+ """
+ Adaptively fits the PCE model by comparing the scores of different
+ degrees and q-norm.
+
+ Parameters
+ ----------
+ ED_X : array of shape (n_samples, n_params)
+ Experimental design.
+ ED_Y : array of shape (n_samples,)
+ Target values, i.e. simulation results for the Experimental design.
+ varIdx : int
+ Index of the output.
+ verbose : bool, optional
+ Print out summary. The default is False.
+
+ Returns
+ -------
+ returnVars : Dict
+ Fitted estimator, best degree, best q-norm, LOOCVScore and
+ coefficients.
+
+ """
+
+ n_samples, n_params = ED_X.shape
+ # Initialization
+ qAllCoeffs, AllCoeffs = {}, {}
+ qAllIndices_Sparse, AllIndices_Sparse = {}, {}
+ qAllclf_poly, Allclf_poly = {}, {}
+ qAllnTerms, AllnTerms = {}, {}
+ qAllLCerror, AllLCerror = {}, {}
+
+ # Extract degree array and qnorm array
+ deg_array = np.array([*self.allBasisIndices], dtype=int)
+ qnorm = [*self.allBasisIndices[str(int(deg_array[0]))]]
+
+ # Some options for EarlyStop
+ errorIncreases = False
+ # Stop degree, if LOO error does not decrease n_checks_degree times
+ n_checks_degree = 3
+ # Stop qNorm, if criterion isn't fulfilled n_checks_qNorm times
+ n_checks_qNorm = 2
+ nqnorms = len(qnorm)
+ qNormEarlyStop = True
+ if nqnorms < n_checks_qNorm+1:
+ qNormEarlyStop = False
+
+ # =====================================================================
+ # basis adaptive polynomial chaos: repeat the calculation by increasing
+ # polynomial degree until the highest accuracy is reached
+ # =====================================================================
+ # For each degree check all q-norms and choose the best one
+ scores = -np.inf * np.ones(deg_array.shape[0])
+ qNormScores = -np.inf * np.ones(nqnorms)
+
+ for degIdx, deg in enumerate(deg_array):
+
+ for qidx, q in enumerate(qnorm):
+
+ # Extract the polynomial basis indices from the pool of
+ # allBasisIndices
+ BasisIndices = self.allBasisIndices[str(deg)][str(q)]
+
+ # Assemble the Psi matrix
+ Psi = self.create_psi(BasisIndices, self.univ_p_val)
+
+ # Calulate the cofficients of the meta model
+ outs = self.regression(Psi, ED_Y, BasisIndices)
+
+ # Calculate and save the score of LOOCV
+ score, LCerror = self.corr_loocv_error(outs['clf_poly'],
+ outs['sparePsi'],
+ outs['coeffs'],
+ ED_Y)
+
+ # Check the convergence of noise for FastARD
+ if self.pce_reg_method == 'FastARD' and \
+ outs['clf_poly'].alpha_ < np.finfo(np.float32).eps:
+ score = -np.inf
+
+ qNormScores[qidx] = score
+ qAllCoeffs[str(qidx+1)] = outs['coeffs']
+ qAllIndices_Sparse[str(qidx+1)] = outs['spareMulti-Index']
+ qAllclf_poly[str(qidx+1)] = outs['clf_poly']
+ qAllnTerms[str(qidx+1)] = BasisIndices.shape[0]
+ qAllLCerror[str(qidx+1)] = LCerror
+
+ # EarlyStop check
+ # if there are at least n_checks_qNorm entries after the
+ # best one, we stop
+ if qNormEarlyStop and \
+ sum(np.isfinite(qNormScores)) > n_checks_qNorm:
+ # If the error has increased the last two iterations, stop!
+ qNormScores_nonInf = qNormScores[np.isfinite(qNormScores)]
+ deltas = np.sign(np.diff(qNormScores_nonInf))
+ if sum(deltas[-n_checks_qNorm+1:]) == 2:
+ # stop the q-norm loop here
+ break
+ if np.var(ED_Y) == 0:
+ break
+
+ # Store the score in the scores list
+ best_q = np.nanargmax(qNormScores)
+ scores[degIdx] = qNormScores[best_q]
+
+ AllCoeffs[str(degIdx+1)] = qAllCoeffs[str(best_q+1)]
+ AllIndices_Sparse[str(degIdx+1)] = qAllIndices_Sparse[str(best_q+1)]
+ Allclf_poly[str(degIdx+1)] = qAllclf_poly[str(best_q+1)]
+ AllnTerms[str(degIdx+1)] = qAllnTerms[str(best_q+1)]
+ AllLCerror[str(degIdx+1)] = qAllLCerror[str(best_q+1)]
+
+ # Check the direction of the error (on average):
+ # if it increases consistently stop the iterations
+ if len(scores[scores != -np.inf]) > n_checks_degree:
+ scores_nonInf = scores[scores != -np.inf]
+ ss = np.sign(scores_nonInf - np.max(scores_nonInf))
+ # ss<0 error decreasing
+ errorIncreases = np.sum(np.sum(ss[-2:])) <= -1*n_checks_degree
+
+ if errorIncreases:
+ break
+
+ # Check only one degree, if target matrix has zero variance
+ if np.var(ED_Y) == 0:
+ break
+
+ # ------------------ Summary of results ------------------
+ # Select the one with the best score and save the necessary outputs
+ best_deg = np.nanargmax(scores)+1
+ coeffs = AllCoeffs[str(best_deg)]
+ basis_indices = AllIndices_Sparse[str(best_deg)]
+ clf_poly = Allclf_poly[str(best_deg)]
+ LOOCVScore = np.nanmax(scores)
+ P = AllnTerms[str(best_deg)]
+ LCerror = AllLCerror[str(best_deg)]
+ degree = deg_array[np.nanargmax(scores)]
+ qnorm = float(qnorm[best_q])
+
+ # ------------------ Print out Summary of results ------------------
+ if self.verbose:
+ # Create PSI_Sparse by removing redundent terms
+ nnz_idx = np.nonzero(coeffs)[0]
+ BasisIndices_Sparse = basis_indices[nnz_idx]
+
+ print(f'Output variable {varIdx+1}:')
+ print('The estimation of PCE coefficients converged at polynomial '
+ f'degree {deg_array[best_deg-1]} with '
+ f'{len(BasisIndices_Sparse)} terms (Sparsity index = '
+ f'{round(len(BasisIndices_Sparse)/P, 3)}).')
+
+ print(f'Final ModLOO error estimate: {1-max(scores):.3e}')
+ print('\n'+'-'*50)
+
+ if verbose:
+ print('='*50)
+ print(' '*10 + ' Summary of results ')
+ print('='*50)
+
+ print("Scores:\n", scores)
+ print("Degree of best score:", self.deg_array[best_deg-1])
+ print("No. of terms:", len(basis_indices))
+ print("Sparsity index:", round(len(basis_indices)/P, 3))
+ print("Best Indices:\n", basis_indices)
+
+ if self.pce_reg_method in ['BRR', 'ARD']:
+ fig, ax = plt.subplots(figsize=(12, 10))
+ plt.title("Marginal log-likelihood")
+ plt.plot(clf_poly.scores_, color='navy', linewidth=2)
+ plt.ylabel("Score")
+ plt.xlabel("Iterations")
+ if self.pce_reg_method.lower() == 'bbr':
+ text = f"$\\alpha={clf_poly.alpha_:.1f}$\n"
+ f"$\\lambda={clf_poly.lambda_:.3f}$\n"
+ f"$L={clf_poly.scores_[-1]:.1f}$"
+ else:
+ text = f"$\\alpha={clf_poly.alpha_:.1f}$\n$"
+ f"\\L={clf_poly.scores_[-1]:.1f}$"
+
+ plt.text(0.75, 0.5, text, fontsize=18, transform=ax.transAxes)
+ plt.show()
+ print('='*80)
+
+ # Create a dict to pass the outputs
+ returnVars = dict()
+ returnVars['clf_poly'] = clf_poly
+ returnVars['degree'] = degree
+ returnVars['qnorm'] = qnorm
+ returnVars['coeffs'] = coeffs
+ returnVars['multi_indices'] = basis_indices
+ returnVars['LOOCVScore'] = LOOCVScore
+ returnVars['LCerror'] = LCerror
+
+ return returnVars
+
+ # -------------------------------------------------------------------------
+ def corr_loocv_error(self, clf, psi, coeffs, y):
+ """
+ Calculates the corrected LOO error for regression on regressor
+ matrix `psi` that generated the coefficients based on [1] and [2].
+
+ [1] Blatman, G., 2009. Adaptive sparse polynomial chaos expansions for
+ uncertainty propagation and sensitivity analysis (Doctoral
+ dissertation, Clermont-Ferrand 2).
+
+ [2] Blatman, G. and Sudret, B., 2011. Adaptive sparse polynomial chaos
+ expansion based on least angle regression. Journal of computational
+ Physics, 230(6), pp.2345-2367.
+
+ Parameters
+ ----------
+ clf : object
+ Fitted estimator.
+ psi : array of shape (n_samples, n_features)
+ The multivariate orthogonal polynomials (regressor).
+ coeffs : array-like of shape (n_features,)
+ Estimated cofficients.
+ y : array of shape (n_samples,)
+ Target values.
+
+ Returns
+ -------
+ R_2 : float
+ LOOCV Validation score (1-LOOCV erro).
+ residual : array of shape (n_samples,)
+ Residual values (y - predicted targets).
+
+ """
+ psi = np.array(psi, dtype=float)
+
+ # Create PSI_Sparse by removing redundent terms
+ nnz_idx = np.nonzero(coeffs)[0]
+ if len(nnz_idx) == 0:
+ nnz_idx = [0]
+ psi_sparse = psi[:, nnz_idx]
+
+ # NrCoeffs of aPCEs
+ P = len(nnz_idx)
+ # NrEvaluation (Size of experimental design)
+ N = psi.shape[0]
+
+ # Build the projection matrix
+ PsiTPsi = np.dot(psi_sparse.T, psi_sparse)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12: #and \
+ # np.linalg.cond(PsiTPsi) < 1/sys.float_info.epsilon:
+ # faster
+ try:
+ M = sp.linalg.solve(PsiTPsi,
+ sp.sparse.eye(PsiTPsi.shape[0]).toarray())
+ except:
+ raise AttributeError('There are too few samples for the corrected loo-cv error. Fit surrogate on at least as many samples as parameters to use this')
+ else:
+ # stabler
+ M = np.linalg.pinv(PsiTPsi)
+
+ # h factor (the full matrix is not calculated explicitly,
+ # only the trace is, to save memory)
+ PsiM = np.dot(psi_sparse, M)
+
+ h = np.sum(np.multiply(PsiM, psi_sparse), axis=1, dtype=np.longdouble)#float128)
+
+ # ------ Calculate Error Loocv for each measurement point ----
+ # Residuals
+ try:
+ residual = clf.predict(psi) - y
+ except:
+ residual = np.dot(psi, coeffs) - y
+
+ # Variance
+ var_y = np.var(y)
+
+ if var_y == 0:
+ norm_emp_error = 0
+ loo_error = 0
+ LCerror = np.zeros((y.shape))
+ return 1-loo_error, LCerror
+ else:
+ norm_emp_error = np.mean(residual**2)/var_y
+
+ # LCerror = np.divide(residual, (1-h))
+ LCerror = residual / (1-h)
+ loo_error = np.mean(np.square(LCerror)) / var_y
+ # if there are NaNs, just return an infinite LOO error (this
+ # happens, e.g., when a strongly underdetermined problem is solved)
+ if np.isnan(loo_error):
+ loo_error = np.inf
+
+ # Corrected Error for over-determined system
+ tr_M = np.trace(M)
+ if tr_M < 0 or abs(tr_M) > 1e6:
+ tr_M = np.trace(np.linalg.pinv(np.dot(psi.T, psi)))
+
+ # Over-determined system of Equation
+ if N > P:
+ T_factor = N/(N-P) * (1 + tr_M)
+
+ # Under-determined system of Equation
+ else:
+ T_factor = np.inf
+
+ corrected_loo_error = loo_error * T_factor
+
+ R_2 = 1 - corrected_loo_error
+
+ return R_2, LCerror
+
+ # -------------------------------------------------------------------------
+ def pca_transformation(self, target, verbose=False):
+ """
+ Transforms the targets (outputs) via Principal Component Analysis
+
+ Parameters
+ ----------
+ target : array of shape (n_samples,)
+ Target values.
+
+ Returns
+ -------
+ pca : obj
+ Fitted sklearnPCA object.
+ OutputMatrix : array of shape (n_samples,)
+ Transformed target values.
+ n_pca_components : int
+ Number of selected principal components.
+
+ """
+ # Transform via Principal Component Analysis
+ if hasattr(self, 'var_pca_threshold'):
+ var_pca_threshold = self.var_pca_threshold
+ else:
+ var_pca_threshold = 100.0
+ n_samples, n_features = target.shape
+
+ if hasattr(self, 'n_pca_components'):
+ n_pca_components = self.n_pca_components
+ else:
+ # Instantiate and fit sklearnPCA object
+ covar_matrix = sklearnPCA(n_components=None)
+ covar_matrix.fit(target)
+ var = np.cumsum(np.round(covar_matrix.explained_variance_ratio_,
+ decimals=5)*100)
+ # Find the number of components to explain self.varPCAThreshold of
+ # variance
+ try:
+ n_components = np.where(var >= var_pca_threshold)[0][0] + 1
+ except IndexError:
+ n_components = min(n_samples, n_features)
+
+ n_pca_components = min(n_samples, n_features, n_components)
+
+ # Print out a report
+ if verbose:
+ print()
+ print('-' * 50)
+ print(f"PCA transformation is performed with {n_pca_components}"
+ " components.")
+ print('-' * 50)
+ print()
+
+ # Fit and transform with the selected number of components
+ pca = sklearnPCA(n_components=n_pca_components, svd_solver='arpack')
+ scaled_target = pca.fit_transform(target)
+
+ return pca, scaled_target, n_pca_components
+
+ # -------------------------------------------------------------------------
+ def gaussian_process_emulator(self, X, y, nug_term=None, autoSelect=False,
+ varIdx=None):
+ """
+ Fits a Gaussian Process Emulator to the target given the training
+ points.
+
+ Parameters
+ ----------
+ X : array of shape (n_samples, n_params)
+ Training points.
+ y : array of shape (n_samples,)
+ Target values.
+ nug_term : float, optional
+ Nugget term. The default is None, i.e. variance of y.
+ autoSelect : bool, optional
+ Loop over some kernels and select the best. The default is False.
+ varIdx : int, optional
+ The index number. The default is None.
+
+ Returns
+ -------
+ gp : object
+ Fitted estimator.
+
+ """
+
+ nug_term = nug_term if nug_term else np.var(y)
+
+ Kernels = [nug_term * kernels.RBF(length_scale=1.0,
+ length_scale_bounds=(1e-25, 1e15)),
+ nug_term * kernels.RationalQuadratic(length_scale=0.2,
+ alpha=1.0),
+ nug_term * kernels.Matern(length_scale=1.0,
+ length_scale_bounds=(1e-15, 1e5),
+ nu=1.5)]
+
+ # Automatic selection of the kernel
+ if autoSelect:
+ gp = {}
+ BME = []
+ for i, kernel in enumerate(Kernels):
+ gp[i] = GaussianProcessRegressor(kernel=kernel,
+ n_restarts_optimizer=3,
+ normalize_y=False)
+
+ # Fit to data using Maximum Likelihood Estimation
+ gp[i].fit(X, y)
+
+ # Store the MLE as BME score
+ BME.append(gp[i].log_marginal_likelihood())
+
+ gp = gp[np.argmax(BME)]
+
+ else:
+ gp = GaussianProcessRegressor(kernel=Kernels[0],
+ n_restarts_optimizer=3,
+ normalize_y=False)
+ gp.fit(X, y)
+
+ # Compute score
+ if varIdx is not None:
+ Score = gp.score(X, y)
+ print('-'*50)
+ print(f'Output variable {varIdx}:')
+ print('The estimation of GPE coefficients converged,')
+ print(f'with the R^2 score: {Score:.3f}')
+ print('-'*50)
+
+ return gp
+
+ # -------------------------------------------------------------------------
+ def eval_metamodel(self, samples):
+ """
+ Evaluates meta-model at the requested samples. One can also generate
+ nsamples.
+
+ Parameters
+ ----------
+ samples : array of shape (n_samples, n_params), optional
+ Samples to evaluate meta-model at. The default is None.
+ nsamples : int, optional
+ Number of samples to generate, if no `samples` is provided. The
+ default is None.
+ sampling_method : str, optional
+ Type of sampling, if no `samples` is provided. The default is
+ 'random'.
+ return_samples : bool, optional
+ Retun samples, if no `samples` is provided. The default is False.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean of the predictions.
+ std_pred : dict
+ Standard deviatioon of the predictions.
+ """
+ # Transform into np array - can also be given as list
+ samples = np.array(samples)
+
+ # Transform samples to the independent space
+ samples = self.InputSpace.transform(
+ samples,
+ method='user'
+ )
+ # Compute univariate bases for the given samples
+ if self.meta_model_type.lower() != 'gpe':
+ univ_p_val = self.univ_basis_vals(
+ samples,
+ n_max=np.max(self.pce_deg)
+ )
+
+ mean_pred_b = {}
+ std_pred_b = {}
+ # Loop over bootstrap iterations
+ for b_i in range(self.n_bootstrap_itrs):
+
+ # Extract model dictionary
+ if self.meta_model_type.lower() == 'gpe':
+ model_dict = self.gp_poly[f'b_{b_i+1}']
+ else:
+ model_dict = self.coeffs_dict[f'b_{b_i+1}']
+
+ # Loop over outputs
+ mean_pred = {}
+ std_pred = {}
+ for output, values in model_dict.items():
+
+ mean = np.empty((len(samples), len(values)))
+ std = np.empty((len(samples), len(values)))
+ idx = 0
+ for in_key, InIdxValues in values.items():
+
+ # Prediction with GPE
+ if self.meta_model_type.lower() == 'gpe':
+ X_T = self.x_scaler[f'b_{b_i+1}'][output].transform(samples)
+ gp = self.gp_poly[f'b_{b_i+1}'][output][in_key]
+ y_mean, y_std = gp.predict(X_T, return_std=True)
+
+ else:
+ # Prediction with PCE
+ # Assemble Psi matrix
+ basis = self.basis_dict[f'b_{b_i+1}'][output][in_key]
+ psi = self.create_psi(basis, univ_p_val)
+
+ # Prediction
+ if self.bootstrap_method != 'fast' or b_i == 0:
+ # with error bar, i.e. use clf_poly
+ clf_poly = self.clf_poly[f'b_{b_i+1}'][output][in_key]
+ try:
+ y_mean, y_std = clf_poly.predict(
+ psi, return_std=True
+ )
+ except TypeError:
+ y_mean = clf_poly.predict(psi)
+ y_std = np.zeros_like(y_mean)
+ else:
+ # without error bar
+ coeffs = self.coeffs_dict[f'b_{b_i+1}'][output][in_key]
+ y_mean = np.dot(psi, coeffs)
+ y_std = np.zeros_like(y_mean)
+
+ mean[:, idx] = y_mean
+ std[:, idx] = y_std
+ idx += 1
+
+ # Save predictions for each output
+ if self.dim_red_method.lower() == 'pca':
+ PCA = self.pca[f'b_{b_i+1}'][output]
+ mean_pred[output] = PCA.inverse_transform(mean)
+ std_pred[output] = np.zeros(mean.shape)
+ else:
+ mean_pred[output] = mean
+ std_pred[output] = std
+
+ # Save predictions for each bootstrap iteration
+ mean_pred_b[b_i] = mean_pred
+ std_pred_b[b_i] = std_pred
+
+ # Change the order of nesting
+ mean_pred_all = {}
+ for i in sorted(mean_pred_b):
+ for k, v in mean_pred_b[i].items():
+ if k not in mean_pred_all:
+ mean_pred_all[k] = [None] * len(mean_pred_b)
+ mean_pred_all[k][i] = v
+
+ # Compute the moments of predictions over the predictions
+ for output in self.out_names:
+ # Only use bootstraps with finite values
+ finite_rows = np.isfinite(
+ mean_pred_all[output]).all(axis=2).all(axis=1)
+ outs = np.asarray(mean_pred_all[output])[finite_rows]
+ # Compute mean
+ mean_pred[output] = np.mean(outs, axis=0)
+ # Compute standard deviation
+ if self.n_bootstrap_itrs > 1:
+ std_pred[output] = np.std(outs, axis=0)
+ else:
+ std_pred[output] = std_pred_b[b_i][output]
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ def create_model_error(self, X, y, Model, name='Calib'):
+ """
+ Fits a GPE-based model error.
+
+ Parameters
+ ----------
+ X : array of shape (n_outputs, n_inputs)
+ Input array. It can contain any forcing inputs or coordinates of
+ extracted data.
+ y : array of shape (n_outputs,)
+ The model response for the MAP parameter set.
+ name : str, optional
+ Calibration or validation. The default is `'Calib'`.
+
+ Returns
+ -------
+ self: object
+ Self object.
+
+ """
+ outputNames = self.out_names
+ self.errorRegMethod = 'GPE'
+ self.errorclf_poly = self.auto_vivification()
+ self.errorScale = self.auto_vivification()
+
+ # Read data
+ # TODO: do this call outside the metamodel
+ MeasuredData = Model.read_observation(case=name)
+
+ # Fitting GPR based bias model
+ for out in outputNames:
+ nan_idx = ~np.isnan(MeasuredData[out])
+ # Select data
+ try:
+ data = MeasuredData[out].values[nan_idx]
+ except AttributeError:
+ data = MeasuredData[out][nan_idx]
+
+ # Prepare the input matrix
+ scaler = MinMaxScaler()
+ delta = data # - y[out][0]
+ BiasInputs = np.hstack((X[out], y[out].reshape(-1, 1)))
+ X_S = scaler.fit_transform(BiasInputs)
+ gp = self.gaussian_process_emulator(X_S, delta)
+
+ self.errorScale[out]["y_1"] = scaler
+ self.errorclf_poly[out]["y_1"] = gp
+
+ return self
+
+ # -------------------------------------------------------------------------
+ def eval_model_error(self, X, y_pred):
+ """
+ Evaluates the error model.
+
+ Parameters
+ ----------
+ X : array
+ Inputs.
+ y_pred : dict
+ Predictions.
+
+ Returns
+ -------
+ mean_pred : dict
+ Mean predition of the GPE-based error model.
+ std_pred : dict
+ standard deviation of the GPE-based error model.
+
+ """
+ mean_pred = {}
+ std_pred = {}
+
+ for Outkey, ValuesDict in self.errorclf_poly.items():
+
+ pred_mean = np.zeros_like(y_pred[Outkey])
+ pred_std = np.zeros_like(y_pred[Outkey])
+
+ for Inkey, InIdxValues in ValuesDict.items():
+
+ gp = self.errorclf_poly[Outkey][Inkey]
+ scaler = self.errorScale[Outkey][Inkey]
+
+ # Transform Samples using scaler
+ for j, pred in enumerate(y_pred[Outkey]):
+ BiasInputs = np.hstack((X[Outkey], pred.reshape(-1, 1)))
+ Samples_S = scaler.transform(BiasInputs)
+ y_hat, y_std = gp.predict(Samples_S, return_std=True)
+ pred_mean[j] = y_hat
+ pred_std[j] = y_std
+ # pred_mean[j] += pred
+
+ mean_pred[Outkey] = pred_mean
+ std_pred[Outkey] = pred_std
+
+ return mean_pred, std_pred
+
+ # -------------------------------------------------------------------------
+ class auto_vivification(dict):
+ """
+ Implementation of perl's AutoVivification feature.
+
+ Source: https://stackoverflow.com/a/651879/18082457
+ """
+
+ def __getitem__(self, item):
+ try:
+ return dict.__getitem__(self, item)
+ except KeyError:
+ value = self[item] = type(self)()
+ return value
+
+ # -------------------------------------------------------------------------
+ def copy_meta_model_opts(self):
+ """
+ This method is a convinient function to copy the metamodel options.
+
+ Returns
+ -------
+ new_MetaModelOpts : object
+ The copied object.
+
+ """
+ # TODO: what properties should be moved to the new object?
+ new_MetaModelOpts = copy.deepcopy(self)
+ new_MetaModelOpts.input_obj = self.input_obj#InputObj
+ new_MetaModelOpts.InputSpace = self.InputSpace
+ #new_MetaModelOpts.InputSpace.meta_Model = 'aPCE'
+ #new_MetaModelOpts.InputSpace.InputObj = self.input_obj
+ #new_MetaModelOpts.InputSpace.ndim = len(self.input_obj.Marginals)
+ new_MetaModelOpts.n_params = len(self.input_obj.Marginals)
+ #new_MetaModelOpts.InputSpace.hdf5_file = None
+
+ return new_MetaModelOpts
+
+ # -------------------------------------------------------------------------
+ def __select_degree(self, ndim, n_samples):
+ """
+ Selects degree based on the number of samples and parameters in the
+ sequential design.
+
+ Parameters
+ ----------
+ ndim : int
+ Dimension of the parameter space.
+ n_samples : int
+ Number of samples.
+
+ Returns
+ -------
+ deg_array: array
+ Array containing the arrays.
+
+ """
+ # Define the deg_array
+ max_deg = np.max(self.pce_deg)
+ min_Deg = np.min(self.pce_deg)
+
+ # TODO: remove the options for sequential?
+ #nitr = n_samples - self.InputSpace.n_init_samples
+
+ # Check q-norm
+ if not np.isscalar(self.pce_q_norm):
+ self.pce_q_norm = np.array(self.pce_q_norm)
+ else:
+ self.pce_q_norm = np.array([self.pce_q_norm])
+
+ def M_uptoMax(maxDeg):
+ n_combo = np.zeros(maxDeg)
+ for i, d in enumerate(range(1, maxDeg+1)):
+ n_combo[i] = math.factorial(ndim+d)
+ n_combo[i] /= math.factorial(ndim) * math.factorial(d)
+ return n_combo
+
+ deg_new = max_deg
+ #d = nitr if nitr != 0 and self.n_params > 5 else 1
+ # d = 1
+ # min_index = np.argmin(abs(M_uptoMax(max_deg)-ndim*n_samples*d))
+ # deg_new = range(1, max_deg+1)[min_index]
+
+ if deg_new > min_Deg and self.pce_reg_method.lower() != 'fastard':
+ deg_array = np.arange(min_Deg, deg_new+1)
+ else:
+ deg_array = np.array([deg_new])
+
+ return deg_array
+
+ def generate_polynomials(self, max_deg=None):
+ # Check for InputSpace
+ if not hasattr(self, 'InputSpace'):
+ raise AttributeError('Generate or add InputSpace before generating polynomials')
+
+ ndim = self.InputSpace.ndim
+ # Create orthogonal polynomial coefficients if necessary
+ if (self.meta_model_type.lower()!='gpe') and max_deg is not None:# and self.input_obj.poly_coeffs_flag:
+ self.polycoeffs = {}
+ for parIdx in tqdm(range(ndim), ascii=True,
+ desc="Computing orth. polynomial coeffs"):
+ poly_coeffs = apoly_construction(
+ self.InputSpace.raw_data[parIdx],
+ max_deg
+ )
+ self.polycoeffs[f'p_{parIdx+1}'] = poly_coeffs
+ else:
+ raise AttributeError('MetaModel cannot generate polynomials in the given scenario!')
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+
+ # Check if its truly a pce-surrogate
+ if self.meta_model_type.lower() == 'gpe':
+ raise AttributeError('Moments can only be computed for pce-type surrogates')
+
+ outputs = self.out_names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(self.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = self.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = self.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = self.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if self.dim_red_method.lower() == 'pca':
+ PCA = self.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = PCA.inverse_transform(np.sqrt(pce_var))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/examples/model-comparison/example_model_comparison.py b/examples/model-comparison/example_model_comparison.py
index ebd80fea82a3caf3ff204b89da96c59737ba502b..e712bba8c370cfb6bdbfd5a42dda64803edab571 100644
--- a/examples/model-comparison/example_model_comparison.py
+++ b/examples/model-comparison/example_model_comparison.py
@@ -280,7 +280,7 @@ if __name__ == "__main__":
# BME Bootstrap optuions
opts_bootstrap = {
"bootstrap": True,
- "n_samples": 10000,
+ "n_samples": 1000,#0,
"Discrepancy": DiscrepancyOpts,
"emulator": True,
"plot_post_pred": False
@@ -289,7 +289,7 @@ if __name__ == "__main__":
# Run model comparison
BayesOpts = BayesModelComparison(
justifiability=True,
- n_bootstarp=100,#00,
+ n_bootstarp=1000,#00,
just_n_meas=2
)
output_dict = BayesOpts.create_model_comparison(
diff --git a/src/bayesvalidrox.egg-info/PKG-INFO b/src/bayesvalidrox.egg-info/PKG-INFO
index dc88c833512df7b831d0849f008eda731b639173..279feebe3b7b4b5a53bdc43208dd65c926f06f2f 100644
--- a/src/bayesvalidrox.egg-info/PKG-INFO
+++ b/src/bayesvalidrox.egg-info/PKG-INFO
@@ -1,10 +1,10 @@
Metadata-Version: 2.1
Name: bayesvalidrox
-Version: 0.0.5
+Version: 1.0.0
Summary: An open-source, object-oriented Python package for surrogate-assisted Bayesain Validation of computational models.
Home-page: https://git.iws.uni-stuttgart.de/inversemodeling/bayesian-validation
-Author: Farid Mohammadi
-Author-email: farid.mohammadi@iws.uni-stuttgart.de
+Author: Farid Mohammadi, Rebecca Kohlhaas
+Author-email: farid.mohammadi@iws.uni-stuttgart.de, rebecca.kohlhaas@iws.uni-stuttgart.de
Classifier: Programming Language :: Python :: 3
Classifier: License :: OSI Approved :: MIT License
Classifier: Operating System :: OS Independent
diff --git a/src/bayesvalidrox.egg-info/SOURCES.txt b/src/bayesvalidrox.egg-info/SOURCES.txt
index d6619704eee21931221fa73b5d2076a2dce99991..344e9840627bb3e5a89593dbd9256472a8ef41d9 100644
--- a/src/bayesvalidrox.egg-info/SOURCES.txt
+++ b/src/bayesvalidrox.egg-info/SOURCES.txt
@@ -29,10 +29,13 @@ src/bayesvalidrox/surrogate_models/exploration.py
src/bayesvalidrox/surrogate_models/glexindex.py
src/bayesvalidrox/surrogate_models/input_space.py
src/bayesvalidrox/surrogate_models/inputs.py
+src/bayesvalidrox/surrogate_models/meta_model_engine.py
src/bayesvalidrox/surrogate_models/orthogonal_matching_pursuit.py
src/bayesvalidrox/surrogate_models/reg_fast_ard.py
src/bayesvalidrox/surrogate_models/reg_fast_laplace.py
+src/bayesvalidrox/surrogate_models/sequential_design.py
src/bayesvalidrox/surrogate_models/surrogate_models.py
+tests/test_BayesModelComparison.py
tests/test_Discrepancy.py
tests/test_ExpDesign.py
tests/test_Input.py
diff --git a/src/bayesvalidrox/bayes_inference/bayes_inference.py b/src/bayesvalidrox/bayes_inference/bayes_inference.py
index 1898a8ae619597d92bc355ac4249f57019f0aed7..44e90b77c69b4c2c7bd27639727c5d2bd4fe8720 100644
--- a/src/bayesvalidrox/bayes_inference/bayes_inference.py
+++ b/src/bayesvalidrox/bayes_inference/bayes_inference.py
@@ -27,6 +27,91 @@ from .mcmc import MCMC
plt.style.use(os.path.join(os.path.split(__file__)[0],
'../', 'bayesvalidrox.mplstyle'))
+# -------------------------------------------------------------------------
+def _kernel_rbf(X, hyperparameters):
+ """
+ Isotropic squared exponential kernel.
+
+ Higher l values lead to smoother functions and therefore to coarser
+ approximations of the training data. Lower l values make functions
+ more wiggly with wide uncertainty regions between training data points.
+
+ sigma_f controls the marginal variance of b(x)
+
+ Parameters
+ ----------
+ X : ndarray of shape (n_samples_X, n_features)
+
+ hyperparameters : Dict
+ Lambda characteristic length
+ sigma_f controls the marginal variance of b(x)
+ sigma_0 unresolvable error nugget term, interpreted as random
+ error that cannot be attributed to measurement error.
+ Returns
+ -------
+ var_cov_matrix : ndarray of shape (n_samples_X,n_samples_X)
+ Kernel k(X, X).
+
+ """
+ from sklearn.gaussian_process.kernels import RBF
+ min_max_scaler = preprocessing.MinMaxScaler()
+ X_minmax = min_max_scaler.fit_transform(X)
+
+ nparams = len(hyperparameters)
+ if nparams <3:
+ raise AttributeError('Provide 3 parameters for the RBF kernel!')
+
+ # characteristic length (0,1]
+ Lambda = hyperparameters[0]
+ # sigma_f controls the marginal variance of b(x)
+ sigma2_f = hyperparameters[1]
+
+ rbf = RBF(length_scale=Lambda)
+ cov_matrix = sigma2_f * rbf(X_minmax)
+
+ # (unresolvable error) nugget term that is interpreted as random
+ # error that cannot be attributed to measurement error.
+ sigma2_0 = hyperparameters[2:]
+ for i, j in np.ndindex(cov_matrix.shape):
+ cov_matrix[i, j] += np.sum(sigma2_0) if i == j else 0
+
+ return cov_matrix
+
+
+# -------------------------------------------------------------------------
+def _logpdf(x, mean, cov):
+ """
+ Computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+
+ # Tranform into np arrays
+ x = np.array(x)
+ mean = np.array(mean)
+ cov = np.array(cov)
+
+ n = len(mean)
+ L = spla.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(spla.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+ return log_lik
+
class BayesInference:
"""
@@ -42,7 +127,7 @@ class BayesInference:
of the variance matrix for a multivariate normal likelihood.
name : str, optional
The type of analysis, either calibration (`Calib`) or validation
- (`Valid`). The default is `'Calib'`.
+ (`Valid`). The default is `'Calib'`. # TODO: what is going on here for validation?
emulator : bool, optional
Analysis with emulator (MetaModel). The default is `True`.
bootstrap : bool, optional
@@ -55,11 +140,11 @@ class BayesInference:
A dictionary with the selected indices of each model output. The
default is `None`. If `None`, all measurement points are used in the
analysis.
- samples : array of shape (n_samples, n_params), optional
+ prior_samples : array of shape (n_samples, n_params), optional
The samples to be used in the analysis. The default is `None`. If
None the samples are drawn from the probablistic input parameter
object of the MetaModel object.
- n_samples : int, optional
+ n_prior_samples : int, optional
Number of samples to be used in the analysis. The default is `500000`.
If samples is not `None`, this argument will be assigned based on the
number of samples given.
@@ -127,13 +212,13 @@ class BayesInference:
def __init__(self, engine, MetaModel = None, discrepancy=None, emulator=True,
name='Calib', bootstrap=False, req_outputs=None,
- selected_indices=None, samples=None, n_samples=100000,
+ selected_indices=None, prior_samples=None, n_prior_samples=100000,
measured_data=None, inference_method='rejection',
mcmc_params=None, bayes_loocv=False, n_bootstrap_itrs=1,
perturbed_data=[], bootstrap_noise=0.05, just_analysis=False,
valid_metrics=['BME'], plot_post_pred=True,
plot_map_pred=False, max_a_posteriori='mean',
- corner_title_fmt='.2e'):
+ corner_title_fmt='.2e', out_dir = ''):
self.engine = engine
self.MetaModel = engine.MetaModel
@@ -143,8 +228,8 @@ class BayesInference:
self.bootstrap = bootstrap
self.req_outputs = req_outputs
self.selected_indices = selected_indices
- self.samples = samples
- self.n_samples = n_samples
+ self.prior_samples = prior_samples
+ self.n_prior_samples = n_prior_samples
self.measured_data = measured_data
self.inference_method = inference_method
self.mcmc_params = mcmc_params
@@ -158,6 +243,7 @@ class BayesInference:
self.plot_map_pred = plot_map_pred
self.max_a_posteriori = max_a_posteriori
self.corner_title_fmt = corner_title_fmt
+ self.out_dir = out_dir
# -------------------------------------------------------------------------
def create_inference(self):
@@ -168,31 +254,37 @@ class BayesInference:
-------
BayesInference : obj
The Bayes inference object.
+
+ # TODO: should this function really return the class?
"""
-
# Set some variables
MetaModel = self.MetaModel
Model = self.engine.Model
n_params = MetaModel.n_params
output_names = Model.Output.names
par_names = self.engine.ExpDesign.par_names
-
- # If the prior is set by the user, take it.
- if self.samples is None:
- self.samples = self.engine.ExpDesign.generate_samples(
- self.n_samples, 'random')
+
+ # Create output directory
+ if self.out_dir == '':
+ self.out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
+ os.makedirs(self.out_dir, exist_ok=True)
+
+ # If the prior is set by the user, take it, else generate from ExpDes
+ if self.prior_samples is None:
+ self.prior_samples = self.engine.ExpDesign.generate_samples(
+ self.n_prior_samples, 'random')
else:
try:
- samples = self.samples.values
+ samples = self.prior_samples.values
except AttributeError:
- samples = self.samples
-
+ samples = self.prior_samples
+
# Take care of an additional Sigma2s
- self.samples = samples[:, :n_params]
-
+ self.prior_samples = samples[:, :self.engine.MetaModel.n_params]
+
# Update number of samples
- self.n_samples = self.samples.shape[0]
+ self.n_prior_samples = self.prior_samples.shape[0]
# ---------- Preparation of observation data ----------
# Read observation data and perturb it if requested.
@@ -229,239 +321,58 @@ class BayesInference:
opt_sigma_flag = isinstance(self.Discrepancy, dict)
opt_sigma = None
for key_idx, key in enumerate(output_names):
-
# Find opt_sigma
if opt_sigma_flag and opt_sigma is None:
# Option A: known error with unknown bias term
opt_sigma = 'A'
- known_discrepancy = self.Discrepancy['known']
- self.Discrepancy = self.Discrepancy['infer']
+ known_discrepancy = self.Discrepancy['known'] # TODO: the syntax here looks different from expected
+ self.Discrepancy = self.Discrepancy['infer'] # TODO: the syntax here looks different from expected
sigma2 = np.array(known_discrepancy.parameters[key])
- elif opt_sigma == 'A' or self.Discrepancy.parameters is not None:
+ elif self.Discrepancy.parameters is not None:
# Option B: The sigma2 is known (no bias term)
- if opt_sigma == 'A':
- sigma2 = np.array(known_discrepancy.parameters[key])
- else:
- opt_sigma = 'B'
- sigma2 = np.array(self.Discrepancy.parameters[key])
+ opt_sigma = 'B'
+ sigma2 = np.array(self.Discrepancy.parameters[key])
elif not isinstance(self.Discrepancy.InputDisc, str):
# Option C: The sigma2 is unknown (bias term including error)
opt_sigma = 'C'
- self.Discrepancy.opt_sigma = opt_sigma
n_measurement = self.measured_data[key].values.shape
sigma2 = np.zeros((n_measurement[0]))
total_sigma2[key] = sigma2
- self.Discrepancy.opt_sigma = opt_sigma
- self.Discrepancy.total_sigma2 = total_sigma2
+ self.Discrepancy.opt_sigma = opt_sigma
+ self.Discrepancy.total_sigma2 = total_sigma2
# If inferred sigma2s obtained from e.g. calibration are given
try:
- self.sigma2s = self.Discrepancy.get_sample(self.n_samples)
+ self.sigma2s = self.Discrepancy.get_sample(self.n_prior_samples)
except:
- pass
+ pass #TODO: should an error be raised in this case?
# ---------------- Bootstrap & TOM --------------------
+
if self.bootstrap or self.bayes_loocv or self.just_analysis:
- if len(self.perturbed_data) == 0:
- # zero mean noise Adding some noise to the observation function
- self.perturbed_data = self._perturb_data(
- self.measured_data, output_names
- )
- else:
- self.n_bootstrap_itrs = len(self.perturbed_data)
-
- # -------- Model Discrepancy -----------
- if hasattr(self, 'error_model') and self.error_model \
- and self.name.lower() != 'calib':
- # Select posterior mean as MAP
- MAP_theta = self.samples.mean(axis=0).reshape((1, n_params))
- # MAP_theta = stats.mode(self.samples,axis=0)[0]
-
- # Evaluate the (meta-)model at the MAP
- y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=MAP_theta)
-
- # Train a GPR meta-model using MAP
- self.error_MetaModel = MetaModel.create_model_error(
- self.bias_inputs, y_MAP, Name=self.name
- )
-
- # -----------------------------------------------------
- # ----- Loop over the perturbed observation data ------
- # -----------------------------------------------------
- # Initilize arrays
- logLikelihoods = np.zeros((self.n_samples, self.n_bootstrap_itrs),
- dtype=np.float16)
- BME_Corr = np.zeros((self.n_bootstrap_itrs))
- log_BME = np.zeros((self.n_bootstrap_itrs))
- KLD = np.zeros((self.n_bootstrap_itrs))
- inf_entropy = np.zeros((self.n_bootstrap_itrs))
-
- # Compute the prior predtions
- # Evaluate the MetaModel
- if self.emulator:
- y_hat, y_std = MetaModel.eval_metamodel(samples=self.samples)
- self.__mean_pce_prior_pred = y_hat
- self._std_pce_prior_pred = y_std
-
- # Correct the predictions with Model discrepancy
- if hasattr(self, 'error_model') and self.error_model:
- y_hat_corr, y_std = self.error_MetaModel.eval_model_error(
- self.bias_inputs, self.__mean_pce_prior_pred
- )
- self.__mean_pce_prior_pred = y_hat_corr
- self._std_pce_prior_pred = y_std
-
- # Surrogate model's error using RMSE of test data
- if hasattr(MetaModel, 'rmse'):
- surrError = MetaModel.rmse
- else:
- surrError = None
-
- else:
- # Evaluate the original model
- self.__model_prior_pred = self._eval_model(
- samples=self.samples, key='PriorPred'
- )
- surrError = None
-
- # Start the likelihood-BME computations for the perturbed data
- for itr_idx, data in tqdm(
- enumerate(self.perturbed_data),
- total=self.n_bootstrap_itrs,
- desc="Bootstrapping the BME calculations", ascii=True
- ):
-
- # ---------------- Likelihood calculation ----------------
- if self.emulator:
- model_evals = self.__mean_pce_prior_pred
- else:
- model_evals = self.__model_prior_pred
-
- # Leave one out
- if self.bayes_loocv or self.just_analysis:
- self.selected_indices = np.nonzero(data)[0]
-
- # Prepare data dataframe
- nobs = list(self.measured_data.count().values[1:])
- numbers = list(np.cumsum(nobs))
- indices = list(zip([0] + numbers, numbers))
- data_dict = {
- output_names[i]: data[j:k] for i, (j, k) in
- enumerate(indices)
- }
- #print(output_names)
- #print(indices)
- #print(numbers)
- #print(nobs)
- #print(self.measured_data)
- #for i, (j, k) in enumerate(indices):
- # print(i,j,k)
- #print(data)
- #print(data_dict)
- #stop
-
- # Unknown sigma2
- if opt_sigma == 'C' or hasattr(self, 'sigma2s'):
- logLikelihoods[:, itr_idx] = self.normpdf(
- model_evals, data_dict, total_sigma2,
- sigma2=self.sigma2s, std=surrError
- )
- else:
- # known sigma2
- logLikelihoods[:, itr_idx] = self.normpdf(
- model_evals, data_dict, total_sigma2,
- std=surrError
- )
-
- # ---------------- BME Calculations ----------------
- # BME (log)
- log_BME[itr_idx] = np.log(
- np.nanmean(np.exp(logLikelihoods[:, itr_idx],
- dtype=np.longdouble))#float128))
- )
-
- # BME correction when using Emulator
- if self.emulator:
- BME_Corr[itr_idx] = self.__corr_factor_BME(
- data_dict, total_sigma2, log_BME[itr_idx]
- )
-
- # Rejection Step
- if 'kld' in list(map(str.lower, self.valid_metrics)) and\
- 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
- # Random numbers between 0 and 1
- unif = np.random.rand(1, self.n_samples)[0]
-
- # Reject the poorly performed prior
- Likelihoods = np.exp(logLikelihoods[:, itr_idx],
- dtype=np.float64)
- accepted = (Likelihoods/np.max(Likelihoods)) >= unif
- posterior = self.samples[accepted]
-
- # Posterior-based expectation of likelihoods
- postExpLikelihoods = np.mean(
- logLikelihoods[:, itr_idx][accepted]
- )
-
- # Calculate Kullback-Leibler Divergence
- KLD[itr_idx] = postExpLikelihoods - log_BME[itr_idx]
-
- # Posterior-based expectation of prior densities
- if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
- n_thread = int(0.875 * multiprocessing.cpu_count())
- with multiprocessing.Pool(n_thread) as p:
- postExpPrior = np.mean(np.concatenate(
- p.map(
- self.engine.ExpDesign.JDist.pdf,
- np.array_split(posterior.T, n_thread, axis=1))
- )
- )
- # Information Entropy based on Entropy paper Eq. 38
- inf_entropy[itr_idx] = log_BME[itr_idx] - postExpPrior - \
- postExpLikelihoods
-
- # Clear memory
- gc.collect(generation=2)
-
- # ---------- Store metrics for perturbed data set ----------------
- # Likelihoods (Size: n_samples, n_bootstrap_itr)
- self.log_likes = logLikelihoods
-
- # BME (log), KLD, infEntropy (Size: 1,n_bootstrap_itr)
- self.log_BME = log_BME
-
- # BMECorrFactor (log) (Size: 1,n_bootstrap_itr)
- if self.emulator:
- self.log_BME_corr_factor = BME_Corr
-
- if 'kld' in list(map(str.lower, self.valid_metrics)):
- self.KLD = KLD
- if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
- self.inf_entropy = inf_entropy
-
- # BME = BME + BMECorrFactor
- if self.emulator:
- self.log_BME += self.log_BME_corr_factor
-
+ self.perform_bootstrap(opt_sigma, total_sigma2)
+ else:
+ print('No bootstrap for TOM performed!') # TODO: stop the code? Use n_bootstrap = 1?
+
# ---------------- Parameter Bayesian inference ----------------
- if self.inference_method.lower() == 'mcmc':
+ if self.name.lower() == 'valid':
+ # Convert to a dataframe if samples are provided after calibration.
+ self.posterior_df = pd.DataFrame(self.prior_samples, columns=par_names)
+ elif self.inference_method.lower() == 'mcmc':
# Instantiate the MCMC object
MCMC_Obj = MCMC(self)
self.posterior_df = MCMC_Obj.run_sampler(
self.measured_data, total_sigma2
)
-
- elif self.name.lower() == 'valid':
- # Convert to a dataframe if samples are provided after calibration.
- self.posterior_df = pd.DataFrame(self.samples, columns=par_names)
-
- else:
+ elif self.inference_method.lower() == 'rejection':
# Rejection sampling
self.posterior_df = self._rejection_sampling()
+ else:
+ raise AttributeError('The chosen inference method is not available!')
# Provide posterior's summary
print('\n')
@@ -502,10 +413,8 @@ class BayesInference:
# ------------------ Visualization --------------------
# -----------------------------------------------------
# Create Output directory, if it doesn't exist already.
- out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
- os.makedirs(out_dir, exist_ok=True)
- # -------- Posteior parameters --------
+ # -------- Posterior parameters --------
if opt_sigma != "B":
par_names.extend(
[self.Discrepancy.InputDisc.Marginals[i].name for i
@@ -549,7 +458,7 @@ class BayesInference:
plotname = f'/Posterior_Dist_{Model.name}'
figPosterior.set_size_inches((24, 16))
- figPosterior.savefig(f'./{out_dir}{plotname}.pdf',
+ figPosterior.savefig(f'./{self.out_dir}{plotname}.pdf',
bbox_inches='tight')
# -------- Plot MAP --------
@@ -558,47 +467,212 @@ class BayesInference:
# -------- Plot log_BME dist --------
if self.bootstrap:
+ self.plot_log_BME()
+
+ # -------- Posterior perdictives --------
+ if self.plot_post_pred:
+ # Plot the posterior predictive
+ self._plot_post_predictive()
+
+ return self
+
+ def perform_bootstrap(self, opt_sigma, total_sigma2):
+ """
+ Perform bootstrap to get TOM (??)
+
+ Parameters
+ ----------
+ opt_sigma : string
+ 'A', 'B', or 'C'
+ total_sigma2 : dict
+ Dictionary containing the sigma2 for the training(?) data
+ Returns
+ -------
+ None.
+
+ """
+ MetaModel = self.MetaModel
+ n_params = MetaModel.n_params
+ output_names = self.engine.Model.Output.names
- # Computing the TOM performance
- self.log_BME_tom = stats.chi2.rvs(
- self.n_tot_measurement, size=self.log_BME.shape[0]
+ # Adding some zero mean noise to the observation function
+ if len(self.perturbed_data) == 0:
+ self.perturbed_data = self._perturb_data(
+ self.measured_data, output_names
)
+ else:
+ self.n_bootstrap_itrs = len(self.perturbed_data)
- fig, ax = plt.subplots()
- sns.kdeplot(self.log_BME_tom, ax=ax, color="green", shade=True)
- sns.kdeplot(
- self.log_BME, ax=ax, color="blue", shade=True,
- label='Model BME')
+ # -------- Model Discrepancy -----------
+ if hasattr(self, 'error_model') and self.error_model \
+ and self.name.lower() != 'calib':# TODO: what should be set so that this is tested?
+ # Select prior ? mean as MAP
+ MAP_theta = self.prior_samples.mean(axis=0).reshape((1, n_params))
- ax.set_xlabel('log$_{10}$(BME)')
- ax.set_ylabel('Probability density')
+ # Evaluate the (meta-)model at the MAP
+ y_MAP, y_std_MAP = MetaModel.eval_metamodel(samples=MAP_theta)
- legend_elements = [
- Patch(facecolor='green', edgecolor='green', label='TOM BME'),
- Patch(facecolor='blue', edgecolor='blue', label='Model BME')
- ]
- ax.legend(handles=legend_elements)
+ # Train a GPR meta-model using MAP
+ self.error_MetaModel = MetaModel.create_model_error(
+ self.bias_inputs, y_MAP, Name=self.name
+ )
- if self.emulator:
- plotname = f'/BME_hist_{Model.name}_emulator'
+ # -----------------------------------------------------
+ # ----- Loop over the perturbed observation data ------
+ # -----------------------------------------------------
+ # Initilize arrays
+ logLikelihoods = np.zeros((self.n_prior_samples, self.n_bootstrap_itrs),
+ dtype=np.float16)
+ BME_Corr = np.zeros((self.n_bootstrap_itrs))
+ log_BME = np.zeros((self.n_bootstrap_itrs))
+ KLD = np.zeros((self.n_bootstrap_itrs))
+ inf_entropy = np.zeros((self.n_bootstrap_itrs))
+
+ # Compute the prior predictions
+ # Evaluate the MetaModel
+ if self.emulator:
+ y_hat, y_std = MetaModel.eval_metamodel(samples=self.prior_samples)
+ self.__mean_pce_prior_pred = y_hat
+ self._std_pce_prior_pred = y_std
+
+ # Correct the predictions with Model discrepancy
+ if hasattr(self, 'error_model') and self.error_model: # TODO this does not check for calib?
+ y_hat_corr, y_std = self.error_MetaModel.eval_model_error(
+ self.bias_inputs, self.__mean_pce_prior_pred
+ )
+ self.__mean_pce_prior_pred = y_hat_corr
+ self._std_pce_prior_pred = y_std
+
+ # Surrogate model's error using RMSE of test data
+ if hasattr(MetaModel, 'rmse'):
+ surrError = MetaModel.rmse
else:
- plotname = f'/BME_hist_{Model.name}'
+ surrError = None
- plt.savefig(f'./{out_dir}{plotname}.pdf', bbox_inches='tight')
- plt.show()
- plt.close()
+ else:
+ # Evaluate the original model
+ self.__model_prior_pred = self._eval_model(
+ samples=self.prior_samples, key='PriorPred'
+ )
+ surrError = None
+
+ # Start the likelihood-BME computations for the perturbed data
+ for itr_idx, data in tqdm(
+ enumerate(self.perturbed_data),
+ total=self.n_bootstrap_itrs,
+ desc="Bootstrapping the BME calculations", ascii=True
+ ):
+ print('')
+ #print(itr_idx, data)
+ #print(np.nonzero(data))
+
+ # ---------------- Likelihood calculation ----------------
+ if self.emulator: # TODO: do this outside of the loop?
+ model_evals = self.__mean_pce_prior_pred
+ else:
+ model_evals = self.__model_prior_pred
+
+ # Leave one out # TODO: why is this loo? It just looks at perturbed data?
+ if self.bayes_loocv or self.just_analysis:
+ # Consider only non-zero entries
+ self.selected_indices = np.nonzero(data)[0]
+
+ # Prepare data dataframe # TODO: what's with this transformation?
+ nobs = list(self.measured_data.count().values[1:])
+ numbers = list(np.cumsum(nobs))
+ indices = list(zip([0] + numbers, numbers))
+ data_dict = {
+ output_names[i]: data[j:k] for i, (j, k) in
+ enumerate(indices)
+ }
- # -------- Posteior perdictives --------
- if self.plot_post_pred:
- # Plot the posterior predictive
- self._plot_post_predictive()
+ # Unknown sigma2
+ if opt_sigma == 'C' or hasattr(self, 'sigma2s'):
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ sigma2=self.sigma2s, std=surrError
+ )
+ else:
+ # known sigma2
+ logLikelihoods[:, itr_idx] = self.normpdf(
+ model_evals, data_dict, total_sigma2,
+ std=surrError
+ )
+ # ---------------- BME Calculations ----------------
+ # BME (log)
+ log_BME[itr_idx] = np.log(
+ np.nanmean(np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.longdouble))#float128))
+ )
+
+ # BME correction when using Emulator
+ if self.emulator:
+ BME_Corr[itr_idx] = self._corr_factor_BME(
+ data_dict, total_sigma2, log_BME[itr_idx]
+ )
+
+ # Rejection Step
+ if 'kld' in list(map(str.lower, self.valid_metrics)) and\
+ 'inf_entropy' in list(map(str.lower, self.valid_metrics)): # TODO: why and and not or?
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, self.n_prior_samples)[0]
+
+ # Reject the poorly performed prior
+ Likelihoods = np.exp(logLikelihoods[:, itr_idx],
+ dtype=np.float64)
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ posterior = self.prior_samples[accepted]
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(
+ logLikelihoods[:, itr_idx][accepted]
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ KLD[itr_idx] = postExpLikelihoods - log_BME[itr_idx]
+
+ # Posterior-based expectation of prior densities
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ n_thread = int(0.875 * multiprocessing.cpu_count())
+ with multiprocessing.Pool(n_thread) as p:
+ postExpPrior = np.mean(np.concatenate(
+ p.map(
+ self.engine.ExpDesign.JDist.pdf,
+ np.array_split(posterior.T, n_thread, axis=1))
+ )
+ )
+ # Information Entropy based on Entropy paper Eq. 38
+ inf_entropy[itr_idx] = log_BME[itr_idx] - postExpPrior - \
+ postExpLikelihoods
+
+ # Clear memory
+ gc.collect(generation=2)
+
+ # ---------- Store metrics for perturbed data set ----------------
+ # Likelihoods (Size: n_samples, n_bootstrap_itr)
+ self.log_likes = logLikelihoods
+
+ # BME (log), KLD, infEntropy (Size: 1,n_bootstrap_itr)
+ self.log_BME = log_BME
+
+ # BMECorrFactor (log) (Size: 1,n_bootstrap_itr)
+ if self.emulator:
+ self.log_BME_corr_factor = BME_Corr
+ # BME = BME + BMECorrFactor
+ self.log_BME += self.log_BME_corr_factor
+
+ if 'kld' in list(map(str.lower, self.valid_metrics)):
+ self.KLD = KLD
+ if 'inf_entropy' in list(map(str.lower, self.valid_metrics)):
+ self.inf_entropy = inf_entropy
+
+
- return self
# -------------------------------------------------------------------------
def _perturb_data(self, data, output_names):
"""
- Returns an array with n_bootstrap_itrs rowsof perturbed data.
+ Returns an array with n_bootstrap_itrs rows of perturbed data.
The first row includes the original observation data.
If `self.bayes_loocv` is True, a 2d-array will be returned with
repeated rows and zero diagonal entries.
@@ -620,12 +694,11 @@ class BayesInference:
obs_data = data[output_names].values
n_measurement, n_outs = obs_data.shape
self.n_tot_measurement = obs_data[~np.isnan(obs_data)].shape[0]
- # Number of bootstrap iterations
- if self.bayes_loocv:
- self.n_bootstrap_itrs = self.n_tot_measurement
-
+
# Pass loocv dataset
if self.bayes_loocv:
+ # Number of bootstrap iterations
+ self.n_bootstrap_itrs = self.n_tot_measurement
obs = obs_data.T[~np.isnan(obs_data.T)]
final_data = np.repeat(np.atleast_2d(obs), self.n_bootstrap_itrs,
axis=0)
@@ -633,6 +706,7 @@ class BayesInference:
return final_data
else:
+ # Init return data with original data
final_data = np.zeros(
(self.n_bootstrap_itrs, self.n_tot_measurement)
)
@@ -640,9 +714,11 @@ class BayesInference:
for itrIdx in range(1, self.n_bootstrap_itrs):
data = np.zeros((n_measurement, n_outs))
for idx in range(len(output_names)):
+ # Perturb the data
std = np.nanstd(obs_data[:, idx])
if std == 0:
- std = 0.001
+ print('Note: Use std=0.01 for perturbation')
+ std = 0.001
noise = std * noise_level
data[:, idx] = np.add(
obs_data[:, idx],
@@ -653,45 +729,18 @@ class BayesInference:
return final_data
- # -------------------------------------------------------------------------
- def _logpdf(self, x, mean, cov):
- """
- computes the likelihood based on a multivariate normal distribution.
-
- Parameters
- ----------
- x : TYPE
- DESCRIPTION.
- mean : array_like
- Observation data.
- cov : 2d array
- Covariance matrix of the distribution.
-
- Returns
- -------
- log_lik : float
- Log likelihood.
-
- """
- n = len(mean)
- L = spla.cholesky(cov, lower=True)
- beta = np.sum(np.log(np.diag(L)))
- dev = x - mean
- alpha = dev.dot(spla.cho_solve((L, True), dev))
- log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
- return log_lik
-
+
# -------------------------------------------------------------------------
def _eval_model(self, samples=None, key='MAP'):
"""
- Evaluates Forward Model.
+ Evaluates Forward Model and zips the results
Parameters
----------
samples : array of shape (n_samples, n_params), optional
Parameter sets. The default is None.
key : str, optional
- Key string to be passed to the run_model_parallel method.
+ Descriptive key string for the run_model_parallel method.
The default is 'MAP'.
Returns
@@ -700,18 +749,17 @@ class BayesInference:
Model outputs.
"""
- MetaModel = self.MetaModel
Model = self.engine.Model
if samples is None:
- self.samples = self.engine.ExpDesign.generate_samples(
- self.n_samples, 'random')
+ self.prior_samples = self.engine.ExpDesign.generate_samples(
+ self.n_prior_samples, 'random')
else:
- self.samples = samples
- self.n_samples = len(samples)
+ self.prior_samples = samples
+ self.n_prior_samples = len(samples)
model_outputs, _ = Model.run_model_parallel(
- self.samples, key_str=key+self.name)
+ self.prior_samples, key_str=key+self.name)
# Clean up
# Zip the subdirectories
@@ -724,55 +772,6 @@ class BayesInference:
return model_outputs
- # -------------------------------------------------------------------------
- def _kernel_rbf(self, X, hyperparameters):
- """
- Isotropic squared exponential kernel.
-
- Higher l values lead to smoother functions and therefore to coarser
- approximations of the training data. Lower l values make functions
- more wiggly with wide uncertainty regions between training data points.
-
- sigma_f controls the marginal variance of b(x)
-
- Parameters
- ----------
- X : ndarray of shape (n_samples_X, n_features)
-
- hyperparameters : Dict
- Lambda characteristic length
- sigma_f controls the marginal variance of b(x)
- sigma_0 unresolvable error nugget term, interpreted as random
- error that cannot be attributed to measurement error.
- Returns
- -------
- var_cov_matrix : ndarray of shape (n_samples_X,n_samples_X)
- Kernel k(X, X).
-
- """
- from sklearn.gaussian_process.kernels import RBF
- min_max_scaler = preprocessing.MinMaxScaler()
- X_minmax = min_max_scaler.fit_transform(X)
-
- nparams = len(hyperparameters)
- # characteristic length (0,1]
- Lambda = hyperparameters[0]
- # sigma_f controls the marginal variance of b(x)
- sigma2_f = hyperparameters[1]
-
- # cov_matrix = sigma2_f*rbf_kernel(X_minmax, gamma = 1/Lambda**2)
-
- rbf = RBF(length_scale=Lambda)
- cov_matrix = sigma2_f * rbf(X_minmax)
- if nparams > 2:
- # (unresolvable error) nugget term that is interpreted as random
- # error that cannot be attributed to measurement error.
- sigma2_0 = hyperparameters[2:]
- for i, j in np.ndindex(cov_matrix.shape):
- cov_matrix[i, j] += np.sum(sigma2_0) if i == j else 0
-
- return cov_matrix
-
# -------------------------------------------------------------------------
def normpdf(self, outputs, obs_data, total_sigma2s, sigma2=None, std=None):
"""
@@ -809,11 +808,11 @@ class BayesInference:
# Extract the requested model outputs for likelihood calulation
if self.req_outputs is None:
- req_outputs = Model.Output.names
+ req_outputs = Model.Output.names # TODO: should this then be saved as self.req_outputs?
else:
req_outputs = list(self.req_outputs)
-
- # Loop over the outputs
+
+ # Loop over the output keys
for idx, out in enumerate(req_outputs):
# (Meta)Model Output
@@ -825,27 +824,26 @@ class BayesInference:
except AttributeError:
data = obs_data[out][~np.isnan(obs_data[out])]
- # Prepare sigma2s
+ # Prepare data uncertainty / error estimation (sigma2s)
non_nan_indices = ~np.isnan(total_sigma2s[out])
tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
- # Add the std of the PCE is chosen as emulator.
+ # Add the std of the PCE if an emulator is used
if self.emulator:
if std is not None:
tot_sigma2s += std[out]**2
- # Covariance Matrix
- covMatrix = np.diag(tot_sigma2s)
-
# Select the data points to compare
try:
indices = self.selected_indices[out]
except:
indices = list(range(nout))
- covMatrix = np.diag(covMatrix[indices, indices])
+
+ # Set up Covariance Matrix
+ covMatrix = np.diag(np.diag(tot_sigma2s)[indices, indices])
- # If sigma2 is not given, use given total_sigma2s
- if sigma2 is None:
+ # If sigma2 is not given, use given total_sigma2s and move to next itr
+ if sigma2 is None:
logLik += stats.multivariate_normal.logpdf(
outputs[out][:, indices], data[indices], covMatrix)
continue
@@ -860,26 +858,25 @@ class BayesInference:
# Covariance Matrix
covMatrix = np.diag(tot_sigma2s)
- if sigma2 is not None:
- # Check the type error term
- if hasattr(self, 'bias_inputs') and \
- not hasattr(self, 'error_model'):
- # Infer a Bias model usig Gaussian Process Regression
- bias_inputs = np.hstack(
- (self.bias_inputs[out],
- tot_outputs[s_idx].reshape(-1, 1)))
-
- params = sigma2[s_idx, idx*3:(idx+1)*3]
- covMatrix = self._kernel_rbf(bias_inputs, params)
- else:
- # Infer equal sigma2s
- try:
- sigma_2 = sigma2[s_idx, idx]
- except TypeError:
- sigma_2 = 0.0
+ # Check the type error term
+ if hasattr(self, 'bias_inputs') and \
+ not hasattr(self, 'error_model'):
+ # Infer a Bias model usig Gaussian Process Regression
+ bias_inputs = np.hstack(
+ (self.bias_inputs[out],
+ tot_outputs[s_idx].reshape(-1, 1)))
+
+ params = sigma2[s_idx, idx*3:(idx+1)*3]
+ covMatrix = _kernel_rbf(bias_inputs, params)
+ else:
+ # Infer equal sigma2s
+ try:
+ sigma_2 = sigma2[s_idx, idx]
+ except TypeError:
+ sigma_2 = 0.0
- covMatrix += sigma_2 * np.eye(nout)
- # covMatrix = np.diag(sigma2 * total_sigma2s)
+ covMatrix += sigma_2 * np.eye(nout)
+ # covMatrix = np.diag(sigma2 * total_sigma2s)
# Select the data points to compare
try:
@@ -889,80 +886,40 @@ class BayesInference:
covMatrix = np.diag(covMatrix[indices, indices])
# Compute loglikelihood
- logliks[s_idx] = self._logpdf(
+ logliks[s_idx] = _logpdf(
tot_outputs[s_idx, indices], data[indices], covMatrix
)
-
+ #print(logLik)
logLik += logliks
return logLik
# -------------------------------------------------------------------------
- def _corr_factor_BME_old(self, Data, total_sigma2s, posterior):
+ def _corr_factor_BME(self, obs_data, total_sigma2s, logBME):
"""
Calculates the correction factor for BMEs.
- """
- MetaModel = self.MetaModel
- OrigModelOutput = self.engine.ExpDesign.Y
- Model = self.engine.Model
-
- # Posterior with guassian-likelihood
- postDist = stats.gaussian_kde(posterior.T)
-
- # Remove NaN
- Data = Data[~np.isnan(Data)]
- total_sigma2s = total_sigma2s[~np.isnan(total_sigma2s)]
-
- # Covariance Matrix
- covMatrix = np.diag(total_sigma2s[:self.n_tot_measurement])
-
- # Extract the requested model outputs for likelihood calulation
- if self.req_outputs is None:
- OutputType = Model.Output.names
- else:
- OutputType = list(self.req_outputs)
-
- # SampleSize = OrigModelOutput[OutputType[0]].shape[0]
-
-
- # Flatten the OutputType for OrigModel
- TotalOutputs = np.concatenate([OrigModelOutput[x] for x in OutputType], 1)
-
- NrofBayesSamples = self.n_samples
- # Evaluate MetaModel on the experimental design
- Samples = self.engine.ExpDesign.X
- OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=Samples)
-
- # Reset the NrofSamples to NrofBayesSamples
- self.n_samples = NrofBayesSamples
-
- # Flatten the OutputType for MetaModel
- TotalPCEOutputs = np.concatenate([OutputRS[x] for x in OutputRS], 1)
- TotalPCEstdOutputRS= np.concatenate([stdOutputRS[x] for x in stdOutputRS], 1)
+
+ Parameters
+ ----------
+ obs_data : dict
+ A dictionary/dataframe containing the observation data.
+ total_sigma2s : dict
+ A dictionary with known values of the covariance diagonal entries,
+ a.k.a sigma^2.
+ logBME : ??
+ ??
- logweight = 0
- for i, sample in enumerate(Samples):
- # Compute likelilhood output vs RS
- covMatrix = np.diag(TotalPCEstdOutputRS[i]**2)
- logLik = self._logpdf(TotalOutputs[i], TotalPCEOutputs[i], covMatrix)
- # Compute posterior likelihood of the collocation points
- logpostLik = np.log(postDist.pdf(sample[:, None]))[0]
- if logpostLik != -np.inf:
- logweight += logLik + logpostLik
- return logweight
+ Returns
+ -------
+ np.log(weights) : ??
+ Correction factors # TODO: factors or log of factors?
- # -------------------------------------------------------------------------
- def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
- """
- Calculates the correction factor for BMEs.
"""
+ # Extract the requested model outputs for likelihood calulation
MetaModel = self.MetaModel
samples = self.engine.ExpDesign.X
model_outputs = self.engine.ExpDesign.Y
- Model = self.engine.Model
n_samples = samples.shape[0]
-
- # Extract the requested model outputs for likelihood calulation
- output_names = Model.Output.names
+ output_names = self.engine.Model.Output.names
# Evaluate MetaModel on the experimental design and ValidSet
OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
@@ -1008,13 +965,13 @@ class BayesInference:
covMatrix_data = np.diag(covMatrix_data[indices, indices])
# Compute likelilhood output vs data
- logLik_data[i] += self._logpdf(
+ logLik_data[i] += _logpdf(
y_m_hat[indices], data[indices],
covMatrix_data
)
# Compute likelilhood output vs surrogate
- logLik_model[i] += self._logpdf(
+ logLik_model[i] += _logpdf(
y_m_hat[indices], y_m[indices],
covMatrix
)
@@ -1037,48 +994,51 @@ class BayesInference:
Posterior samples of the input parameters.
"""
+ if self.prior_samples is None:
+ raise AttributeError('No prior samples available!')
- MetaModel = self.MetaModel
+ if not hasattr(self, 'log_likes'):
+ raise AttributeError('No log-likelihoods available!')
+
+ # Get sigmas # TODO: is this data uncertainty?
try:
sigma2_prior = self.Discrepancy.sigma2_prior
except:
sigma2_prior = None
- # Check if the discrepancy is defined as a distribution:
- samples = self.samples
-
+ # Combine samples and sigma2 for the return
+ samples = self.prior_samples
if sigma2_prior is not None:
samples = np.hstack((samples, sigma2_prior))
# Take the first column of Likelihoods (Observation data without noise)
if self.just_analysis or self.bayes_loocv:
index = self.n_tot_measurement-1
- likelihoods = np.exp(self.log_likes[:, index], dtype=np.longdouble)#np.float128)
else:
- likelihoods = np.exp(self.log_likes[:, 0], dtype=np.longdouble)#np.float128)
+ index = 0
+
+ # Use longdouble on windows, float128 on linux
+ if os.name != 'nt':
+ likelihoods = np.exp(self.log_likes[:, index], dtype=np.float128)
+ else:
+ print('WARNING: Performing the inference on windows can lead to reduced accuracy!')
+ likelihoods = np.exp(self.log_likes[:, index], dtype=np.longdouble)
n_samples = len(likelihoods)
- norm_ikelihoods = likelihoods / np.max(likelihoods)
+ norm_likelihoods = likelihoods / np.max(likelihoods)
# Normalize based on min if all Likelihoods are zero
if all(likelihoods == 0.0):
likelihoods = self.log_likes[:, 0]
- norm_ikelihoods = likelihoods / np.min(likelihoods)
+ norm_likelihoods = likelihoods / np.min(likelihoods)
- # Random numbers between 0 and 1
+ # Reject the poorly performed prior compared to a uniform distribution
unif = np.random.rand(1, n_samples)[0]
-
- # Reject the poorly performed prior
- accepted_samples = samples[norm_ikelihoods >= unif]
-
- # Output the Posterior
- par_names = self.engine.ExpDesign.par_names
- if sigma2_prior is not None:
- for name in self.Discrepancy.name:
- par_names.append(name)
+ accepted_samples = samples[norm_likelihoods >= unif]
return pd.DataFrame(accepted_samples, columns=sigma2_prior)
+
# -------------------------------------------------------------------------
def _posterior_predictive(self):
"""
@@ -1100,11 +1060,7 @@ class BayesInference:
MetaModel = self.MetaModel
Model = self.engine.Model
- # Make a directory to save the prior/posterior predictive
- out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
- os.makedirs(out_dir, exist_ok=True)
-
- # Read observation data and perturb it if requested
+ # Read observation data and perturb it if requested # TODO: where is the perturbation?
if self.measured_data is None:
self.measured_data = Model.read_observation(case=self.name)
@@ -1125,7 +1081,7 @@ class BayesInference:
# Take care of the sigma2
if sigma2_prior is not None:
try:
- sigma2s = posterior_df[self.Discrepancy.name].values
+ sigma2s = posterior_df[self.Discrepancy.name].values # TODO: what is Discrepancy.name?
posterior_df = posterior_df.drop(
labels=self.Discrepancy.name, axis=1
)
@@ -1134,17 +1090,17 @@ class BayesInference:
# Posterior predictive
if self.emulator:
- if self.inference_method == 'rejection':
+ if self.inference_method == 'rejection': # TODO: combine these two?
prior_pred = self.__mean_pce_prior_pred
if self.name.lower() != 'calib':
post_pred = self.__mean_pce_prior_pred
post_pred_std = self._std_pce_prior_pred
else:
- post_pred, post_pred_std = MetaModel.eval_metamodel(
+ post_pred, post_pred_std = MetaModel.eval_metamodel( # TODO: recheck if this is needed
samples=posterior_df.values
)
- else:
+ else: # TODO: see emulator version
if self.inference_method == 'rejection':
prior_pred = self.__model_prior_pred
if self.name.lower() != 'calib':
@@ -1182,7 +1138,7 @@ class BayesInference:
bias_inputs = np.hstack((
self.bias_inputs[var], pred.reshape(-1, 1)))
params = sigma2s[i, varIdx*3:(varIdx+1)*3]
- cov = self._kernel_rbf(bias_inputs, params)
+ cov = _kernel_rbf(bias_inputs, params)
else:
# Infer equal sigma2s
try:
@@ -1211,7 +1167,7 @@ class BayesInference:
# ----- Prior Predictive -----
if self.inference_method.lower() == 'rejection':
# Create hdf5 metadata
- hdf5file = f'{out_dir}/priorPredictive.hdf5'
+ hdf5file = f'{self.out_dir}/priorPredictive.hdf5'
hdf5_exist = os.path.exists(hdf5file)
if hdf5_exist:
os.remove(hdf5file)
@@ -1232,7 +1188,7 @@ class BayesInference:
# ----- Posterior Predictive only model evaluations -----
# Create hdf5 metadata
- hdf5file = out_dir+'/postPredictive_wo_noise.hdf5'
+ hdf5file = self.out_dir+'/postPredictive_wo_noise.hdf5'
hdf5_exist = os.path.exists(hdf5file)
if hdf5_exist:
os.remove(hdf5file)
@@ -1253,7 +1209,7 @@ class BayesInference:
# ----- Posterior Predictive with noise -----
# Create hdf5 metadata
- hdf5file = out_dir+'/postPredictive.hdf5'
+ hdf5file = self.out_dir+'/postPredictive.hdf5'
hdf5_exist = os.path.exists(hdf5file)
if hdf5_exist:
os.remove(hdf5file)
@@ -1288,7 +1244,6 @@ class BayesInference:
MetaModel = self.MetaModel
Model = self.engine.Model
- out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
opt_sigma = self.Discrepancy.opt_sigma
# -------- Find MAP and run MetaModel and origModel --------
@@ -1324,7 +1279,7 @@ class BayesInference:
Marker = 'x'
# Create a PdfPages object
- pdf = PdfPages(f'./{out_dir}MAP_PCE_vs_Model_{self.name}.pdf')
+ pdf = PdfPages(f'./{self.out_dir}MAP_PCE_vs_Model_{self.name}.pdf')
fig = plt.figure()
for i, key in enumerate(Model.Output.names):
@@ -1374,6 +1329,38 @@ class BayesInference:
plt.clf()
pdf.close()
+
+ def plot_log_BME(self):
+
+ # Computing the TOM performance
+ self.log_BME_tom = stats.chi2.rvs(
+ self.n_tot_measurement, size=self.log_BME.shape[0]
+ )
+
+ fig, ax = plt.subplots()
+ sns.kdeplot(self.log_BME_tom, ax=ax, color="green", shade=True)
+ sns.kdeplot(
+ self.log_BME, ax=ax, color="blue", shade=True,
+ label='Model BME')
+
+ ax.set_xlabel('log$_{10}$(BME)')
+ ax.set_ylabel('Probability density')
+
+ legend_elements = [
+ Patch(facecolor='green', edgecolor='green', label='TOM BME'),
+ Patch(facecolor='blue', edgecolor='blue', label='Model BME')
+ ]
+ ax.legend(handles=legend_elements)
+
+ if self.emulator:
+ plotname = f'/BME_hist_{self.Model.name}_emulator'
+ else:
+ plotname = f'/BME_hist_{self.Model.name}'
+
+ plt.savefig(f'./{self.self.out_dir}{plotname}.pdf', bbox_inches='tight')
+ plt.show()
+ plt.close()
+
# -------------------------------------------------------------------------
def _plot_post_predictive(self):
@@ -1387,7 +1374,6 @@ class BayesInference:
"""
Model = self.engine.Model
- out_dir = f'Outputs_Bayes_{Model.name}_{self.name}'
# Plot the posterior predictive
for out_idx, out_name in enumerate(Model.Output.names):
fig, ax = plt.subplots()
@@ -1400,7 +1386,7 @@ class BayesInference:
# --- Prior ---
# Load posterior predictive
f = h5py.File(
- f'{out_dir}/priorPredictive.hdf5', 'r+')
+ f'{self.out_dir}/priorPredictive.hdf5', 'r+')
try:
x_coords = np.array(f[f"x_values/{out_name}"])
@@ -1422,7 +1408,7 @@ class BayesInference:
f.close()
# --- Posterior ---
- f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+ f = h5py.File(f"{self.out_dir}/postPredictive.hdf5", 'r+')
X_values = np.repeat(
x_coords, np.array(f[f"EDY/{out_name}"]).shape[0])
@@ -1477,7 +1463,7 @@ class BayesInference:
else:
# Load posterior predictive
- f = h5py.File(f"{out_dir}/postPredictive.hdf5", 'r+')
+ f = h5py.File(f"{self.out_dir}/postPredictive.hdf5", 'r+')
try:
x_coords = np.array(f[f"x_values/{out_name}"])
@@ -1528,5 +1514,5 @@ class BayesInference:
else:
plotname = f'/Post_Prior_Perd_{Model.name}'
- fig.savefig(f'./{out_dir}{plotname}_{out_name}.pdf',
+ fig.savefig(f'./{self.out_dir}{plotname}_{out_name}.pdf',
bbox_inches='tight')
diff --git a/src/bayesvalidrox/bayes_inference/bayes_model_comparison.py b/src/bayesvalidrox/bayes_inference/bayes_model_comparison.py
index 828613556e90ec0c529b91f2592eec148c98136b..769ad2ceaaced2c1fb6f18d22a9ca27278c3e8a1 100644
--- a/src/bayesvalidrox/bayes_inference/bayes_model_comparison.py
+++ b/src/bayesvalidrox/bayes_inference/bayes_model_comparison.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
+import emcee
import numpy as np
import os
from scipy import stats
@@ -27,7 +28,7 @@ class BayesModelComparison:
`True`.
perturbed_data : array of shape (n_bootstrap_itrs, n_obs), optional
User defined perturbed data. The default is `None`.
- n_bootstarp : int
+ n_bootstrap : int
Number of bootstrap iteration. The default is `1000`.
data_noise_level : float
A noise level to perturb the data set. The default is `0.01`.
@@ -38,13 +39,15 @@ class BayesModelComparison:
"""
def __init__(self, justifiability=True, perturbed_data=None,
- n_bootstarp=1000, data_noise_level=0.01, just_n_meas=2):
+ n_bootstrap=1000, data_noise_level=0.01, just_n_meas=2):
+ # TODO: check valid ranges of the parameters
+
self.justifiability = justifiability
self.perturbed_data = perturbed_data
- self.n_bootstarp = n_bootstarp
+ self.n_bootstrap = n_bootstrap
self.data_noise_level = data_noise_level
- self.just_n_meas = just_n_meas
+ self.just_n_meas = just_n_meas # TODO: what is this parameter?
# --------------------------------------------------------------------------
def create_model_comparison(self, model_dict, opts_dict):
@@ -77,7 +80,9 @@ class BayesModelComparison:
comparison using Bayes factors and justifiability analysis.
"""
-
+ # TODO: why are these two separate calls of the same function?
+ # They should be performable at the same time
+
# Bayes factor
bayes_dict_bf, model_weights_dict_bf = self.compare_models(
model_dict, opts_dict
@@ -93,7 +98,8 @@ class BayesModelComparison:
bayes_dict_ja, model_weights_dict_ja = self.compare_models(
model_dict, opts_dict, justifiability=True
)
-
+
+ # TODO: why does this version of the call not return a summarized confusion matrix?
output['Bayes objects JA'] = bayes_dict_ja
output['Model weights JA'] = model_weights_dict_ja
@@ -135,6 +141,7 @@ class BayesModelComparison:
self.model_names = [*model_dict]
# Compute total number of the measurement points
+ # TODO: there could be a different option for this here
Engine = list(model_dict.items())[0][1]
Engine.Model.read_observation()
self.n_meas = Engine.Model.n_obs
@@ -142,21 +149,40 @@ class BayesModelComparison:
# ----- Generate data -----
# Find n_bootstrap
if self.perturbed_data is None:
- n_bootstarp = self.n_bootstarp
+ n_bootstrap = self.n_bootstrap
else:
- n_bootstarp = self.perturbed_data.shape[0]
+ n_bootstrap = self.perturbed_data.shape[0]
# Create dataset
justData = self.generate_dataset(
- model_dict, justifiability, n_bootstarp=n_bootstarp)
+ model_dict, justifiability, n_bootstarp=n_bootstrap)
# Run create Interface for each model
- bayes_dict = {}
+ self.bayes_dict = {}
for model in model_dict.keys():
print("-"*20)
print("Bayesian inference of {}.\n".format(model))
BayesOpts = BayesInference(model_dict[model])
+
+ # Explicitly set the settings of the BayesOpts
+ if self.use_Bayes_settings:
+ BayesOpts.emulator= True
+ BayesOpts.plot_post_pred = True
+ #BayesOpts.inference_method = 'rejection'
+ BayesOpts.bootstrap = True
+ BayesOpts.n_bootstrap_itrs = 10
+ BayesOpts.bootstrap_noise = 0.05
+
+ # Set the MCMC parameters
+ BayesOpts.inference_method = "MCMC"
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e3,#5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'multiprocessing': False,
+ 'verbose': False
+ }
# Set BayesInference options
for key, value in opts_dict.items():
@@ -170,17 +196,17 @@ class BayesModelComparison:
BayesOpts.perturbed_data = justData
BayesOpts.just_analysis = justifiability
- bayes_dict[model] = BayesOpts.create_inference()
+ self.bayes_dict[model] = BayesOpts.create_inference()
print("-"*20)
# Compute model weights
- BME_Dict = dict()
- for modelName, bayesObj in bayes_dict.items():
- BME_Dict[modelName] = np.exp(bayesObj.log_BME, dtype=np.longdouble)#float128)
+ self.BME_Dict = dict()
+ for modelName, bayesObj in self.bayes_dict.items():
+ self.BME_Dict[modelName] = np.exp(bayesObj.log_BME, dtype=np.longdouble)#float128)
# BME correction in BayesInference class
- model_weights = self.cal_model_weight(
- BME_Dict, justifiability, n_bootstarp=n_bootstarp)
+ self.model_weights = self.cal_model_weight(
+ self.BME_Dict, justifiability, n_bootstarp=n_bootstrap)
# Plot model weights
if justifiability:
@@ -189,27 +215,27 @@ class BayesModelComparison:
# Split the model weights and save in a dict
list_ModelWeights = np.split(
- model_weights, model_weights.shape[1]/self.n_meas, axis=1)
+ self.model_weights, self.model_weights.shape[1]/self.n_meas, axis=1)
model_weights_dict = {key: weights for key, weights in
zip(model_names, list_ModelWeights)}
#self.plot_just_analysis(model_weights_dict)
else:
# Create box plot for model weights
- self.plot_model_weights(model_weights, 'model_weights')
+ self.plot_model_weights(self.model_weights, 'model_weights')
# Create kde plot for bayes factors
- self.plot_bayes_factor(BME_Dict, 'kde_plot')
+ self.plot_bayes_factor(self.BME_Dict, 'kde_plot')
# Store model weights in a dict
model_weights_dict = {key: weights for key, weights in
- zip(self.model_names, model_weights)}
+ zip(self.model_names, self.model_weights)}
- return bayes_dict, model_weights_dict
+ return self.bayes_dict, model_weights_dict
# -------------------------------------------------------------------------
def generate_dataset(self, model_dict, justifiability=False,
- n_bootstarp=1):
+ n_bootstrap=1):
"""
Generates the perturbed data set for the Bayes factor calculations and
the data set for the justifiability analysis.
@@ -221,7 +247,7 @@ class BayesModelComparison:
bool, optional
Whether to perform the justifiability analysis. The default is
`False`.
- n_bootstarp : int, optional
+ n_bootstrap : int, optional
Number of bootstrap iterations. The default is `1`.
Returns
@@ -238,7 +264,7 @@ class BayesModelComparison:
# Perturb observations for Bayes Factor
if self.perturbed_data is None:
self.perturbed_data = self.__perturb_data(
- Engine.Model.observations, out_names, n_bootstarp,
+ Engine.Model.observations, out_names, n_bootstrap,
noise_level=self.data_noise_level)
# Only for Bayes Factor
@@ -248,11 +274,11 @@ class BayesModelComparison:
# Evaluate metamodel
runs = {}
for key, metaModel in model_dict.items():
- y_hat, _ = metaModel.eval_metamodel(nsamples=n_bootstarp)
+ y_hat, _ = metaModel.eval_metamodel(nsamples=n_bootstrap)
runs[key] = y_hat
# Generate data
- for i in range(n_bootstarp):
+ for i in range(n_bootstrap):
y_data = self.perturbed_data[i].reshape(1, -1)
justData = np.tril(np.repeat(y_data, y_data.shape[1], axis=0))
# Use surrogate runs for data-generating process
@@ -318,7 +344,7 @@ class BayesModelComparison:
return final_data
# -------------------------------------------------------------------------
- def cal_model_weight(self, BME_Dict, justifiability=False, n_bootstarp=1):
+ def cal_model_weight(self, BME_Dict, justifiability=False, n_bootstrap=1):
"""
Normalize the BME (Asumption: Model Prior weights are equal for models)
@@ -339,7 +365,7 @@ class BayesModelComparison:
if justifiability:
# Compute expected log_BME for justifiabiliy analysis
all_BME = all_BME.reshape(
- all_BME.shape[0], -1, n_bootstarp).mean(axis=2)
+ all_BME.shape[0], -1, n_bootstrap).mean(axis=2)
# Model weights
model_weights = np.divide(all_BME, np.nansum(all_BME, axis=0))
diff --git a/src/bayesvalidrox/bayes_inference/discrepancy.py b/src/bayesvalidrox/bayes_inference/discrepancy.py
index fff32a2500ae20b3667c7b0ec2cc85c1da614688..b3c235ebeb6d6ae9e109ca862cc522cc21efb45e 100644
--- a/src/bayesvalidrox/bayes_inference/discrepancy.py
+++ b/src/bayesvalidrox/bayes_inference/discrepancy.py
@@ -36,7 +36,7 @@ class Discrepancy:
* Option B: With unknown redidual covariance matrix \\(\\Sigma\\),
paramethrized as \\(\\Sigma(\\theta_{\\epsilon})=\\sigma^2 \\textbf{I}_
{N_{out}}\\) with unknown residual variances \\(\\sigma^2\\).
- This term will be jointly infer with the uncertain input parameters. For
+ This term will be jointly infered with the uncertain input parameters. For
the inversion, you need to define a prior marginal via `Input` class. Note
that \\(\\sigma^2\\) is only a single scalar multiplier for the diagonal
entries of the covariance matrix \\(\\Sigma\\).
@@ -58,10 +58,17 @@ class Discrepancy:
"""
def __init__(self, InputDisc='', disc_type='Gaussian', parameters=None):
+ # Set the values
self.InputDisc = InputDisc
self.disc_type = disc_type
self.parameters = parameters
-
+
+ # Other inits
+ self.ExpDesign = None
+ self.n_samples = None
+ self.sigma2_prior = None
+ self.name = None
+ self.opt_sigma = None # This will be set in the inference class and used in mcmc
# -------------------------------------------------------------------------
def get_sample(self, n_samples):
"""
@@ -87,6 +94,11 @@ class Discrepancy:
# Create and store BoundTuples
self.ExpDesign = ExpDesigns(self.InputDisc)
self.ExpDesign.sampling_method = 'random'
+
+ # TODO: why does it call 'generate_ED' instead of 'generate_samples?
+ # ExpDesign.bound_tuples, onp_sigma, prior_space needed from the outside
+ # Discrepancy opt_sigma, InputDisc needed from the outside
+ # TODO: opt_sigma not defined here, but called from the outside??
self.ExpDesign.generate_ED(
n_samples, max_pce_deg=1
)
diff --git a/src/bayesvalidrox/bayes_inference/mcmc.py b/src/bayesvalidrox/bayes_inference/mcmc.py
index fe22a152f117aab7023bfe6592ce3a48bb0b3aec..d78d15b5fd90dc4477da7d0fd58da835acc75310 100755
--- a/src/bayesvalidrox/bayes_inference/mcmc.py
+++ b/src/bayesvalidrox/bayes_inference/mcmc.py
@@ -99,7 +99,7 @@ class MCMC:
initsamples = priorDist.sample(self.nwalkers).T
except:
# when aPCE selected - gaussian kernel distribution
- inputSamples = MetaModel.ExpDesign.raw_data.T
+ inputSamples = self.BayesOpts.engine.ExpDesign.raw_data.T
random_indices = np.random.choice(
len(inputSamples), size=self.nwalkers, replace=False
)
diff --git a/src/bayesvalidrox/surrogate_models/desktop.ini b/src/bayesvalidrox/surrogate_models/desktop.ini
new file mode 100644
index 0000000000000000000000000000000000000000..632de13ae6b61cecf0d9fdbf9c97cfb16bfb51a4
--- /dev/null
+++ b/src/bayesvalidrox/surrogate_models/desktop.ini
@@ -0,0 +1,2 @@
+[LocalizedFileNames]
+exploration.py=@exploration.py,0
diff --git a/src/bayesvalidrox/surrogate_models/engine.py b/src/bayesvalidrox/surrogate_models/engine.py
index 42307d4770d4ae23a40107dfea64057aac682c23..387cec5010373a087b01e838aba89404f2069c51 100644
--- a/src/bayesvalidrox/surrogate_models/engine.py
+++ b/src/bayesvalidrox/surrogate_models/engine.py
@@ -143,6 +143,7 @@ class Engine():
self.Model = Model
self.ExpDesign = ExpDes
self.parallel = False
+ self.trained = False
def start_engine(self) -> None:
"""
@@ -225,6 +226,9 @@ class Engine():
self.ExpDesign.sampling_method.lower() != 'user':
self.Model.zip_subdirs(self.Model.name, f'{self.Model.name}_')
+ # Set that training was done
+ self.trained = True
+
def train_sequential(self, parallel = False, verbose = False) -> None:
"""
@@ -353,7 +357,8 @@ class Engine():
TotalSigma2 = {}
# ---------- Initial self.MetaModel ----------
- self.train_normal(parallel = parallel, verbose=verbose)
+ if not self.trained:
+ self.train_normal(parallel = parallel, verbose=verbose)
initMetaModel = deepcopy(self.MetaModel)
@@ -745,11 +750,14 @@ class Engine():
logPriorLikelihoods = np.zeros((mc_size))
# print(y_hat)
# print(list[y_hat])
+ #print(std)
for key in list(y_hat):
+ #print(std[key])
cov = np.diag(std[key]**2)
- # print(y_hat[key], cov)
+ #print(y_hat[key], cov)
+ print(key, y_hat[key], std[key])
# TODO: added the allow_singular = True here
- rv = stats.multivariate_normal(mean=y_hat[key], cov=cov,)
+ rv = stats.multivariate_normal(mean=y_hat[key], cov=cov,allow_singular = True)
Y_MC[key] = rv.rvs(size=mc_size)
logPriorLikelihoods += rv.logpdf(Y_MC[key])
std_MC[key] = np.zeros((mc_size, y_hat[key].shape[0]))
diff --git a/src/bayesvalidrox/surrogate_models/exp_designs.py b/src/bayesvalidrox/surrogate_models/exp_designs.py
index fa03fe17d96fb2c1f19546b7b72fb2fd6dd1c13a..96012162a614c308c18ba09c8c57344e442f9c43 100644
--- a/src/bayesvalidrox/surrogate_models/exp_designs.py
+++ b/src/bayesvalidrox/surrogate_models/exp_designs.py
@@ -371,10 +371,18 @@ class ExpDesigns(InputSpace):
# store the raw data with given random indices
samples[:, pa_idx] = self.raw_data[pa_idx, rand_idx]
else:
+ if not hasattr(self, 'JDist'):
+ raise AttributeError('Sampling cannot proceed, build ExpDesign with max_deg != 0 to create JDist!')
try:
+ # Use resample if JDist is of type gaussian_kde
samples = self.JDist.resample(int(n_samples)).T
except AttributeError:
+ # Use sample if JDist is of type chaospy.J
samples = self.JDist.sample(int(n_samples)).T
+ # If there is only one input transform the samples
+ if self.ndim == 1:
+ samples = np.swapaxes(np.atleast_2d(samples),0,1)
+
# Check if all samples are in the bound_tuples
for idx, param_set in enumerate(samples):
if not check_ranges(param_set, self.bound_tuples):
diff --git a/src/bayesvalidrox/surrogate_models/input_space.py b/src/bayesvalidrox/surrogate_models/input_space.py
index 4e010d66f2933ec243bad756d8f2c5454808d802..d722e387a8106a07472f8612aaaa7f2fe8daea84 100644
--- a/src/bayesvalidrox/surrogate_models/input_space.py
+++ b/src/bayesvalidrox/surrogate_models/input_space.py
@@ -100,8 +100,6 @@ class InputSpace:
up_bound = np.max(Inputs.Marginals[i].input_data)
Inputs.Marginals[i].parameters = [low_bound, up_bound]
-
-
# -------------------------------------------------------------------------
def init_param_space(self, max_deg=None):
"""
@@ -386,6 +384,8 @@ class InputSpace:
if params == None:
raise AttributeError('Additional parameters have to be set for the gamma distribution!')
params_Y = [1, params[1]]
+
+ # TOOD: update the call to the gamma function, seems like source code has been changed!
dist_Y = st.gamma(loc=params_Y[0], scale=params_Y[1])
inv_cdf = np.vectorize(lambda x: dist_Y.ppf(x))
diff --git a/src/bayesvalidrox/surrogate_models/meta_model_engine.py b/src/bayesvalidrox/surrogate_models/meta_model_engine.py
new file mode 100644
index 0000000000000000000000000000000000000000..71c0244216b0c87a22174a3ad2043a4c0a80efab
--- /dev/null
+++ b/src/bayesvalidrox/surrogate_models/meta_model_engine.py
@@ -0,0 +1,2195 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 28 09:21:18 2022
+
+@author: farid
+"""
+import numpy as np
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from copy import deepcopy, copy
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import sys
+import os
+import gc
+import seaborn as sns
+from joblib import Parallel, delayed
+
+import bayesvalidrox
+from .exploration import Exploration
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+from bayesvalidrox.bayes_inference.discrepancy import Discrepancy
+import pandas as pd
+
+
+class MetaModelEngine():
+ """ Sequential experimental design
+ This class provieds method for trainig the meta-model in an iterative
+ manners.
+ The main method to execute the task is `train_seq_design`, which
+ recieves a model object and returns the trained metamodel.
+ """
+
+ def __init__(self, meta_model_opts):
+ self.MetaModel = meta_model_opts
+
+ # -------------------------------------------------------------------------
+ def run(self):
+
+ Model = self.MetaModel.ModelObj
+ self.MetaModel.n_params = len(self.MetaModel.input_obj.Marginals)
+ self.MetaModel.ExpDesignFlag = 'normal'
+ # --- Prepare pce degree ---
+ if self.MetaModel.meta_model_type.lower() == 'pce':
+ if type(self.MetaModel.pce_deg) is not np.ndarray:
+ self.MetaModel.pce_deg = np.array(self.MetaModel.pce_deg)
+
+ if self.MetaModel.ExpDesign.method == 'normal':
+ self.MetaModel.ExpDesignFlag = 'normal'
+ self.MetaModel.train_norm_design(parallel = False)
+
+ elif self.MetaModel.ExpDesign.method == 'sequential':
+ self.train_seq_design()
+ else:
+ raise Exception("The method for experimental design you requested"
+ " has not been implemented yet.")
+
+ # Zip the model run directories
+ if self.MetaModel.ModelObj.link_type.lower() == 'pylink' and\
+ self.MetaModel.ExpDesign.sampling_method.lower() != 'user':
+ Model.zip_subdirs(Model.name, f'{Model.name}_')
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ # Set model to have shorter call
+ Model = self.MetaModel.ModelObj
+ # MetaModel = self.MetaModel
+ self.Model = Model
+
+ # Initialization
+ self.MetaModel.SeqModifiedLOO = {}
+ self.MetaModel.seqValidError = {}
+ self.MetaModel.SeqBME = {}
+ self.MetaModel.SeqKLD = {}
+ self.MetaModel.SeqDistHellinger = {}
+ self.MetaModel.seqRMSEMean = {}
+ self.MetaModel.seqRMSEStd = {}
+ self.MetaModel.seqMinDist = []
+
+ # Determine the metamodel type
+ if self.MetaModel.meta_model_type.lower() != 'gpe':
+ pce = True
+ else:
+ pce = False
+ # If given, use mc reference data
+ mc_ref = True if bool(Model.mc_reference) else False
+ if mc_ref:
+ Model.read_mc_reference()
+
+ # if valid_samples not defined, do so now
+ if not hasattr(self.MetaModel, 'valid_samples'):
+ self.MetaModel.valid_samples = []
+ self.MetaModel.valid_model_runs = []
+ self.MetaModel.valid_likelihoods = []
+
+ # Get the parameters
+ max_n_samples = self.MetaModel.ExpDesign.n_max_samples
+ mod_LOO_threshold = self.MetaModel.ExpDesign.mod_LOO_threshold
+ n_canddidate = self.MetaModel.ExpDesign.n_canddidate
+ post_snapshot = self.MetaModel.ExpDesign.post_snapshot
+ n_replication = self.MetaModel.ExpDesign.n_replication
+ util_func = self.MetaModel.ExpDesign.util_func
+ output_name = Model.Output.names
+ validError = None
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [self.MetaModel.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ if len(Model.observations) != 0 or Model.meas_file is not None:
+ self.observations = Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ TotalSigma2 = {}
+
+ # TODO: ---------- Initial self.MetaModel ----------
+ # First run MetaModel on non-sequential design
+ self.MetaModel.train_norm_design(parallel = False)
+ initMetaModel = deepcopy(self.MetaModel)
+
+ # Validation error if validation set is provided. - use as initial errors
+ if self.MetaModel.valid_model_runs:
+ init_rmse, init_valid_error = self.__validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(self.MetaModel, 'Discrepancy'):
+ TotalSigma2 = self.MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self.__BME_Calculator(
+ initMetaModel, obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = self.MetaModel.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self.__error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean:.2f},"
+ f" {init_rmse_std:.2f}")
+
+ # Read the initial experimental design
+ # TODO: this sequential, or the non-sequential samples??
+ Xinit = initMetaModel.ExpDesign.X
+ init_n_samples = len(initMetaModel.ExpDesign.X)
+ initYprev = initMetaModel.ModelOutputDict
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ # Replicate the sequential design
+ for repIdx in range(n_replication): # TODO: what does this do?
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # To avoid changes ub original aPCE object
+ self.MetaModel.ExpDesign.X = Xinit
+ self.MetaModel.ExpDesign.Y = initYprev
+ self.MetaModel.LCerror = initLCerror
+
+ for util_f in util_func: # TODO: recheck choices for this
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ self.MetaModel.ExpDesign.X = Xinit
+ self.MetaModel.ExpDesign.Y = initYprev
+ self.MetaModel.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(self.MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(self.MetaModel) # Write last MetaModel here
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel( # What's the use of this here??
+ samples=Xfull[-1].reshape(1, -1))
+ del prevMetaModel_dict[itr_no-1] # Delete second to last metamodel here?
+
+ # Optimal Bayesian Design
+ self.MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.opt_SeqDesign(TotalSigma2, # TODO: check in this!!
+ n_canddidate,
+ util_f)
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ self.MetaModel.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(self.MetaModel, 'adapt_verbose') and \
+ self.MetaModel.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = self.MetaModel.eval_metamodel(
+ samples=Xnew
+ )
+ adaptPlot(
+ self.MetaModel, Ynew, y_hat, std_hat,
+ plotED=False
+ )
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ self.MetaModel.ModelOutputDict[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ self.MetaModel.ExpDesign.sampling_method = 'user'
+ self.MetaModel.ExpDesign.X = Xfull
+ self.MetaModel.ExpDesign.Y = self.MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = self.MetaModel.ModelOutputDict
+
+ # Pass the new prior as the input
+ self.MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ self.MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ self.MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ self.MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ self.MetaModel.train_norm_design(parallel=False)
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = self.MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ self.MetaModel.score_dict['b_1'][out_name].values()))
+ if self.MetaModel.dim_red_method.lower() == 'pca':
+ pca = self.MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if self.MetaModel.valid_model_runs:
+ rmse, validError = self.__validError(self.MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(self.MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self.__BME_Calculator(self.MetaModel, obs_data,
+ TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = self.MetaModel.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = self.MetaModel.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self.__error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ self.MetaModel.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(self.MetaModel.valid_model_runs) != 0:
+ self.MetaModel.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ self.MetaModel.SeqBME[strKey] = SeqBME
+ self.MetaModel.SeqKLD[strKey] = SeqKLD
+ if hasattr(self.MetaModel, 'valid_likelihoods') and \
+ self.MetaModel.valid_likelihoods:
+ self.MetaModel.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ self.MetaModel.seqRMSEMean[strKey] = seqRMSEMean
+ self.MetaModel.seqRMSEStd[strKey] = seqRMSEStd
+
+ # return self.MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = MetaModel.ExpDesign.X
+ out_dict_y = MetaModel.ExpDesign.Y
+ out_names = MetaModel.ModelObj.Output.names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, y_hat, std, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Get the data
+ obs_data = self.observations
+ n_obs = self.Model.n_obs
+ mc_size = 10000
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((mc_size))
+ for key in list(y_hat):
+ cov = np.diag(std[key]**2)
+ rv = stats.multivariate_normal(mean=y_hat[key], cov=cov)
+ Y_MC[key] = rv.rvs(size=mc_size)
+ logPriorLikelihoods += rv.logpdf(Y_MC[key])
+ std_MC[key] = np.zeros((mc_size, y_hat[key].shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, mc_size)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods), dtype=np.longdouble)
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = np.nanmean(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self.__normpdf(
+ y_hat, std, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def update_metamodel(self, MetaModel, output, y_hat_can, univ_p_val, index,
+ new_pca=False):
+ BasisIndices = MetaModel.basis_dict[output]["y_"+str(index+1)]
+ clf_poly = MetaModel.clf_poly[output]["y_"+str(index+1)]
+ Mn = clf_poly.coef_
+ Sn = clf_poly.sigma_
+ beta = clf_poly.alpha_
+ active = clf_poly.active_
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ Sn_new_inv = np.linalg.inv(Sn)
+ Sn_new_inv += beta * np.dot(Psi[:, active].T, Psi[:, active])
+ Sn_new = np.linalg.inv(Sn_new_inv)
+
+ Mn_new = np.dot(Sn_new_inv, Mn[active]).reshape(-1, 1)
+ Mn_new += beta * np.dot(Psi[:, active].T, y_hat_can)
+ Mn_new = np.dot(Sn_new, Mn_new).flatten()
+
+ # Compute the old and new moments of PCEs
+ mean_old = Mn[0]
+ mean_new = Mn_new[0]
+ std_old = np.sqrt(np.sum(np.square(Mn[1:])))
+ std_new = np.sqrt(np.sum(np.square(Mn_new[1:])))
+
+ # Back transformation if PCA is selected.
+ if MetaModel.dim_red_method.lower() == 'pca':
+ old_pca = MetaModel.pca[output]
+ mean_old = old_pca.mean_[index]
+ mean_old += np.sum(mean_old * old_pca.components_[:, index])
+ std_old = np.sqrt(np.sum(std_old**2 *
+ old_pca.components_[:, index]**2))
+ mean_new = new_pca.mean_[index]
+ mean_new += np.sum(mean_new * new_pca.components_[:, index])
+ std_new = np.sqrt(np.sum(std_new**2 *
+ new_pca.components_[:, index]**2))
+ # print(f"mean_old: {mean_old:.2f} mean_new: {mean_new:.2f}")
+ # print(f"std_old: {std_old:.2f} std_new: {std_new:.2f}")
+ # Store the old and new moments of PCEs
+ results = {
+ 'mean_old': mean_old,
+ 'mean_new': mean_new,
+ 'std_old': std_old,
+ 'std_new': std_new
+ }
+ return results
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign_old(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ old_EDY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels using the candidate design
+ Y_PC_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Generate y from posterior predictive
+ m_size = 100
+ y_hat_samples = {}
+ for idx, key in enumerate(Model.Output.names):
+ means, stds = Y_PC_can[key][0], Y_std_can[key][0]
+ y_hat_samples[key] = np.random.multivariate_normal(
+ means, np.diag(stds), m_size)
+
+ # Create the SparseBayes-based PCE metamodel:
+ MetaModel.input_obj.poly_coeffs_flag = False
+ univ_p_val = self.MetaModel.univ_basis_vals(X_can)
+ G_n_m_all = np.zeros((m_size, len(Model.Output.names), Model.n_obs))
+
+ for i in range(m_size):
+ for idx, key in enumerate(Model.Output.names):
+ if MetaModel.dim_red_method.lower() == 'pca':
+ # Equal number of components
+ new_outputs = np.vstack(
+ (old_EDY[key], y_hat_samples[key][i])
+ )
+ new_pca, _ = MetaModel.pca_transformation(new_outputs)
+ target = new_pca.transform(
+ y_hat_samples[key][i].reshape(1, -1)
+ )[0]
+ else:
+ new_pca, target = False, y_hat_samples[key][i]
+
+ for j in range(len(target)):
+
+ # Update surrogate
+ result = self.update_metamodel(
+ MetaModel, key, target[j], univ_p_val, j, new_pca)
+
+ # Compute Expected Information Gain (Eq. 39)
+ G_n_m = np.log(result['std_old']/result['std_new']) - 1./2
+ G_n_m += result['std_new']**2 / (2*result['std_old']**2)
+ G_n_m += (result['mean_new'] - result['mean_old'])**2 /\
+ (2*result['std_old']**2)
+
+ G_n_m_all[i, idx, j] = G_n_m
+
+ U_J_d = G_n_m_all.mean(axis=(1, 2)).mean()
+ return -1 * U_J_d
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ MetaModel = self.MetaModel
+ out_names = MetaModel.ModelObj.Output.names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+ oldExpDesignY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+ PCE_Model_can = deepcopy(MetaModel)
+ # Add the candidate to the ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ NewExpDesignY[key] = np.vstack(
+ (oldExpDesignY[key], Y_PC_can[key])
+ )
+
+ PCE_Model_can.ExpDesign.sampling_method = 'user'
+ PCE_Model_can.ExpDesign.X = NewExpDesignX
+ PCE_Model_can.ModelOutputDict = NewExpDesignY
+ PCE_Model_can.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ PCE_Model_can.input_obj.poly_coeffs_flag = False
+ PCE_Model_can.train_norm_design(parallel=False)
+
+ # Set the ExpDesign to its original values
+ PCE_Model_can.ExpDesign.X = oldExpDesignX
+ PCE_Model_can.ModelOutputDict = oldExpDesignY
+ PCE_Model_can.ExpDesign.Y = oldExpDesignY
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = PCE_Model_can.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.MetaModel.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.longdouble))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postLikelihoods /= np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def subdomain(self, Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ Subdomains : TYPE
+ DESCRIPTION.
+
+ """
+ n_params = self.MetaModel.n_params
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ # Evaluate all candidates
+ y_can, std_can = self.MetaModel.eval_metamodel(samples=candidates)
+ # loop through candidates
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="BAL Design"):
+ y_hat = {key: items[idx] for key, items in y_can.items()}
+ std = {key: items[idx] for key, items in std_can.items()}
+ U_J_d[idx] = self.util_BayesianActiveDesign(
+ y_hat, std, sigma2Dict, var)
+
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithim to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.MetaModel.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def opt_SeqDesign(self, sigma2, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None.
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ MetaModel = self.MetaModel
+ Bounds = MetaModel.bound_tuples
+ n_new_samples = MetaModel.ExpDesign.n_new_samples
+ explore_method = MetaModel.ExpDesign.explore_method
+ exploit_method = MetaModel.ExpDesign.exploit_method
+ n_cand_groups = MetaModel.ExpDesign.n_cand_groups
+ tradeoff_scheme = MetaModel.ExpDesign.tradeoff_scheme
+
+ old_EDX = MetaModel.ExpDesign.X
+ old_EDY = MetaModel.ExpDesign.Y.copy()
+ ndim = MetaModel.ExpDesign.X.shape[1]
+ OutputNames = MetaModel.ModelObj.Output.names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+
+ Xnew, filteredSamples = MetaModel.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ args = []
+ subdomains = self.subdomain(Bounds, n_new_samples)
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+
+ print("\nXnew:\n", Xnew)
+
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ elif explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = MetaModel.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(MetaModel, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = MetaModel.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = MetaModel.ExpDesign.generate_samples(
+ MetaModel.ExpDesign.max_func_itr, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+
+ results = Parallel(n_jobs=-1, backend='multiprocessing')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ # out = map(self.run_util_func,
+ # [exploit_method]*n_cand_groups,
+ # split_cand,
+ # range(n_cand_groups),
+ # [sigma2] * n_cand_groups,
+ # [var] * n_cand_groups,
+ # [X_MC] * n_cand_groups
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # create surrogate model for U_J_d
+ # from sklearn.preprocessing import MinMaxScaler
+ # # Take care of inf entries
+ # good_indices = [i for i, arr in enumerate(U_J_d)
+ # if np.isfinite(arr).all()]
+ # scaler = MinMaxScaler()
+ # X_S = scaler.fit_transform(candidates[good_indices])
+ # gp = MetaModel.gaussian_process_emulator(
+ # X_S, U_J_d[good_indices], autoSelect=False
+ # )
+ # U_J_d = gp.predict(scaler.transform(allCandidates))
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Accumulate the samples
+ # TODO: added this, recheck!!
+ finalCandidates = np.concatenate((allCandidates, candidates), axis = 0)
+ finalCandidates = np.unique(finalCandidates, axis = 0)
+
+ #self.allCandidates = allCandidates
+ #self.candidates = candidates
+ #self.norm_U_J_d = norm_U_J_d
+ #self.exploit_w = exploit_w
+ #self.scoreExploration = scoreExploration
+
+ # Total score
+ #totalScore = exploit_w * norm_U_J_d
+ #totalScore += explore_w * scoreExploration
+
+ # TODO: changed this from the above to take into account both exploration and exploitation samples without duplicates
+ totalScore = np.zeros(finalCandidates.shape[0])
+ #self.totalScore = totalScore
+
+ for cand_idx in range(finalCandidates.shape[0]):
+ # find candidate indices
+ idx1 = np.where(allCandidates == finalCandidates[cand_idx])[0]
+ idx2 = np.where(candidates == finalCandidates[cand_idx])[0]
+ #print(f'Candidate number {cand_idx}')
+ #print(finalCandidates[cand_idx])
+ #print(f'Idx1: {idx1}, Idx2: {idx2}')
+
+ # exploration
+ if idx1 != []:
+ idx1 = idx1[0]
+ #print(f'Values1: {allCandidates[idx1]}')
+ totalScore[cand_idx] += explore_w * scoreExploration[idx1]
+
+ # exploitation
+ if idx2 != []:
+ idx2 = idx2[0]
+ #print(f'Values1: {candidates[idx2]}')
+ totalScore[cand_idx] += exploit_w * norm_U_J_d[idx2]
+
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ # TODO: changed this from allCandiates to full set of candidates - still not changed for e.g. 'Voronoi'
+ Xnew = finalCandidates[sorted_idxtotalScore[:n_new_samples]] # here candidates(exploitation) vs allCandidates (exploration)!!
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+ # out = map(self.run_util_func,
+ # [exploit_method]*len(goodSampleIdx),
+ # split_cand,
+ # range(len(goodSampleIdx)),
+ # [sigma2] * len(goodSampleIdx),
+ # [var] * len(goodSampleIdx)
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self
+ Model = self.Model
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = Model.Output.names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ Scores = list(MetaModel.score_dict[OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ BasisIndices = MetaModelOrig.basis_dict[OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = MetaModelOrig.univ_basis_vals(oldExpDesignX)
+ Psi = MetaModelOrig.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.univ_basis_vals(candidates)
+ Psi_c = self.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var == 'D-Opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var == 'A-Opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var == 'K-Opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def __normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+
+ Model = self.Model
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(Model.Output.names):
+
+ # (Meta)Model Output
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+ else:
+ tot_sigma2s += np.mean(std_pce[out])**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = MetaModel.ExpDesign.X # valid_samples
+ model_outputs = MetaModel.ExpDesign.Y # valid_model_runs
+ Model = MetaModel.ModelObj
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self.__logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self.__logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def __logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def __posteriorPlot(self, posterior, par_names, key):
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.MetaModel.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+ # -------------------------------------------------------------------------
+ def __hellinger_distance(self, P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+ # -------------------------------------------------------------------------
+ def __BME_Calculator(self, MetaModel, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ """
+ # Initializations
+ if hasattr(MetaModel, 'valid_likelihoods'):
+ valid_likelihoods = MetaModel.valid_likelihoods
+ else:
+ valid_likelihoods = []
+
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ #print(f'post_snapshot: {post_snapshot}')
+ if post_snapshot or len(valid_likelihoods) != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = MetaModel.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ Y_MC, std_MC = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self.__normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(MetaModel.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or valid_likelihoods:
+ # Hellinger distance
+ #print('arrived here')
+ #print(np.array(valid_likelihoods))
+ valid_likelihoods = np.array(valid_likelihoods)
+ #valid_likelihoods = np.array(valid_likelihoods)
+ ref_like = np.log(valid_likelihoods[(valid_likelihoods > 0)])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = self.__hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and MetaModel.n_params == 2 and not idx % 5:
+ BayesOpts = BayesInference(MetaModel)
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ obs_data = pd.DataFrame(obs_data, columns=self.Model.Output.names)
+ BayesOpts.measurement_error = obs_data
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def __validError(self, MetaModel):
+
+ # MetaModel = self.MetaModel
+ Model = MetaModel.ModelObj
+ OutputName = Model.Output.names
+
+ # Extract the original model with the generated samples
+ valid_samples = MetaModel.valid_samples
+ valid_model_runs = MetaModel.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ valid_PCE_runs, _ = MetaModel.eval_metamodel(samples=valid_samples)
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in OutputName:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def __error_Mean_Std(self):
+
+ MetaModel = self.MetaModel
+ # Extract the mean and std provided by user
+ df_MCReference = MetaModel.ModelObj.mc_reference
+
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+
+ # Compute the root mean squared error
+ for output in MetaModel.ModelObj.Output.names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ df_MCReference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ df_MCReference['std'], pce_means[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self, MetaModel):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+ outputs = MetaModel.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.inverse_transform(pce_mean)
+ stds[output] = PCA.inverse_transform(np.sqrt(pce_var))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/src/bayesvalidrox/surrogate_models/sequential_design.py b/src/bayesvalidrox/surrogate_models/sequential_design.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc81dcd4529ca0708dfba47385aef4415992eb3e
--- /dev/null
+++ b/src/bayesvalidrox/surrogate_models/sequential_design.py
@@ -0,0 +1,2187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jan 28 09:21:18 2022
+
+@author: farid
+"""
+import numpy as np
+from scipy import stats, signal, linalg, sparse
+from scipy.spatial import distance
+from copy import deepcopy, copy
+from tqdm import tqdm
+import scipy.optimize as opt
+from sklearn.metrics import mean_squared_error
+import multiprocessing
+import matplotlib.pyplot as plt
+import sys
+import os
+import gc
+import seaborn as sns
+from joblib import Parallel, delayed
+import resource
+from .exploration import Exploration
+
+
+class SeqDesign():
+ """ Sequential experimental design
+ This class provieds method for trainig the meta-model in an iterative
+ manners.
+ The main method to execute the task is `train_seq_design`, which
+ recieves a model object and returns the trained metamodel.
+ """
+
+ # -------------------------------------------------------------------------
+ def train_seq_design(self, MetaModel):
+ """
+ Starts the adaptive sequential design for refining the surrogate model
+ by selecting training points in a sequential manner.
+
+ Parameters
+ ----------
+ Model : object
+ An object containing all model specifications.
+
+ Returns
+ -------
+ MetaModel : object
+ Meta model object.
+
+ """
+ # MetaModel = self
+ Model = MetaModel.ModelObj
+ self.MetaModel = MetaModel
+ self.Model = Model
+
+ # Initialization
+ MetaModel.SeqModifiedLOO = {}
+ MetaModel.seqValidError = {}
+ MetaModel.SeqBME = {}
+ MetaModel.SeqKLD = {}
+ MetaModel.SeqDistHellinger = {}
+ MetaModel.seqRMSEMean = {}
+ MetaModel.seqRMSEStd = {}
+ MetaModel.seqMinDist = []
+ pce = True if MetaModel.meta_model_type.lower() != 'gpe' else False
+ mc_ref = True if bool(Model.mc_reference) else False
+ if mc_ref:
+ Model.read_mc_reference()
+
+ if not hasattr(MetaModel, 'valid_likelihoods'):
+ MetaModel.valid_samples = []
+ MetaModel.valid_model_runs = []
+ MetaModel.valid_likelihoods = []
+
+ # Get the parameters
+ max_n_samples = MetaModel.ExpDesign.n_max_samples
+ mod_LOO_threshold = MetaModel.ExpDesign.mod_LOO_threshold
+ n_canddidate = MetaModel.ExpDesign.n_canddidate
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ n_replication = MetaModel.ExpDesign.n_replication
+ util_func = MetaModel.ExpDesign.util_func
+ output_name = Model.Output.names
+ validError = None
+ # Handle if only one UtilityFunctions is provided
+ if not isinstance(util_func, list):
+ util_func = [MetaModel.ExpDesign.util_func]
+
+ # Read observations or MCReference
+ if len(Model.observations) != 0 or Model.meas_file is not None:
+ self.observations = Model.read_observation()
+ obs_data = self.observations
+ else:
+ obs_data = []
+ TotalSigma2 = {}
+ # ---------- Initial MetaModel ----------
+ initMetaModel = deepcopy(MetaModel)
+
+ # Validation error if validation set is provided.
+ if len(MetaModel.valid_model_runs) != 0:
+ init_rmse, init_valid_error = self.__validError(initMetaModel)
+ init_valid_error = list(init_valid_error.values())
+ else:
+ init_rmse = None
+
+ # Check if discrepancy is provided
+ if len(obs_data) != 0 and hasattr(MetaModel, 'Discrepancy'):
+ TotalSigma2 = MetaModel.Discrepancy.parameters
+
+ # Calculate the initial BME
+ out = self.__BME_Calculator(
+ initMetaModel, obs_data, TotalSigma2, init_rmse)
+ init_BME, init_KLD, init_post, init_likes, init_dist_hellinger = out
+ print(f"\nInitial BME: {init_BME:.2f}")
+ print(f"Initial KLD: {init_KLD:.2f}")
+
+ # Posterior snapshot (initial)
+ if post_snapshot:
+ parNames = MetaModel.ExpDesign.par_names
+ print('Posterior snapshot (initial) is being plotted...')
+ self.__posteriorPlot(init_post, parNames, 'SeqPosterior_init')
+
+ # Check the convergence of the Mean & Std
+ if mc_ref and pce:
+ init_rmse_mean, init_rmse_std = self.__error_Mean_Std()
+ print(f"Initial Mean and Std error: {init_rmse_mean},"
+ f" {init_rmse_std}")
+
+ # Read the initial experimental design
+ Xinit = initMetaModel.ExpDesign.X
+ init_n_samples = len(MetaModel.ExpDesign.X)
+ initYprev = initMetaModel.ModelOutputDict
+ initLCerror = initMetaModel.LCerror
+ n_itrs = max_n_samples - init_n_samples
+
+ # Read the initial ModifiedLOO
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = initMetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ initMetaModel.score_dict['b_1'][out_name].values()))
+ if MetaModel.dim_red_method.lower() == 'pca':
+ pca = MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components, axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+
+ Scores = [item for sublist in Scores_all for item in sublist]
+ weights = [item for sublist in varExpDesignY for item in sublist]
+ init_mod_LOO = [np.average([1-score for score in Scores],
+ weights=weights)]
+
+ prevMetaModel_dict = {}
+ # Replicate the sequential design
+ for repIdx in range(n_replication):
+ print(f'\n>>>> Replication: {repIdx+1}<<<<')
+
+ # To avoid changes ub original aPCE object
+ MetaModel.ExpDesign.X = Xinit
+ MetaModel.ExpDesign.Y = initYprev
+ MetaModel.LCerror = initLCerror
+
+ for util_f in util_func:
+ print(f'\n>>>> Utility Function: {util_f} <<<<')
+ # To avoid changes ub original aPCE object
+ MetaModel.ExpDesign.X = Xinit
+ MetaModel.ExpDesign.Y = initYprev
+ MetaModel.LCerror = initLCerror
+
+ # Set the experimental design
+ Xprev = Xinit
+ total_n_samples = init_n_samples
+ Yprev = initYprev
+
+ Xfull = []
+ Yfull = []
+
+ # Store the initial ModifiedLOO
+ if pce:
+ print("\nInitial ModifiedLOO:", init_mod_LOO)
+ SeqModifiedLOO = np.array(init_mod_LOO)
+
+ if len(MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.array(init_valid_error)
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.array([init_BME])
+ SeqKLD = np.array([init_KLD])
+ SeqDistHellinger = np.array([init_dist_hellinger])
+
+ if mc_ref and pce:
+ seqRMSEMean = np.array([init_rmse_mean])
+ seqRMSEStd = np.array([init_rmse_std])
+
+ # ------- Start Sequential Experimental Design -------
+ postcnt = 1
+ for itr_no in range(1, n_itrs+1):
+ print(f'\n>>>> Iteration number {itr_no} <<<<')
+
+ # Save the metamodel prediction before updating
+ prevMetaModel_dict[itr_no] = deepcopy(MetaModel)
+ if itr_no > 1:
+ pc_model = prevMetaModel_dict[itr_no-1]
+ self._y_hat_prev, _ = pc_model.eval_metamodel(
+ samples=Xfull[-1].reshape(1, -1))
+
+ # Optimal Bayesian Design
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ MetaModel.ExpDesignFlag = 'sequential'
+ Xnew, updatedPrior = self.opt_SeqDesign(TotalSigma2,
+ n_canddidate,
+ util_f)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ S = np.min(distance.cdist(Xinit, Xnew, 'euclidean'))
+ MetaModel.seqMinDist.append(S)
+ print(f"\nmin Dist from OldExpDesign: {S:2f}")
+ print("\n")
+
+ # Evaluate the full model response at the new sample
+ Ynew, _ = Model.run_model_parallel(
+ Xnew, prevRun_No=total_n_samples
+ )
+ total_n_samples += Xnew.shape[0]
+ # ------ Plot the surrogate model vs Origninal Model ------
+ if hasattr(MetaModel, 'adapt_verbose') and \
+ MetaModel.adapt_verbose:
+ from .adaptPlot import adaptPlot
+ y_hat, std_hat = MetaModel.eval_metamodel(samples=Xnew)
+ adaptPlot(MetaModel, Ynew, y_hat, std_hat, plotED=False)
+
+ # -------- Retrain the surrogate model -------
+ # Extend new experimental design
+ Xfull = np.vstack((Xprev, Xnew))
+
+ # Updating experimental design Y
+ for out_name in output_name:
+ Yfull = np.vstack((Yprev[out_name], Ynew[out_name]))
+ MetaModel.ModelOutputDict[out_name] = Yfull
+
+ # Pass new design to the metamodel object
+ MetaModel.ExpDesign.sampling_method = 'user'
+ MetaModel.ExpDesign.X = Xfull
+ MetaModel.ExpDesign.Y = MetaModel.ModelOutputDict
+
+ # Save the Experimental Design for next iteration
+ Xprev = Xfull
+ Yprev = MetaModel.ModelOutputDict
+
+ # Pass the new prior as the input
+ MetaModel.input_obj.poly_coeffs_flag = False
+ if updatedPrior is not None:
+ MetaModel.input_obj.poly_coeffs_flag = True
+ print("updatedPrior:", updatedPrior.shape)
+ # Arbitrary polynomial chaos
+ for i in range(updatedPrior.shape[1]):
+ MetaModel.input_obj.Marginals[i].dist_type = None
+ x = updatedPrior[:, i]
+ MetaModel.input_obj.Marginals[i].raw_data = x
+
+ # Train the surrogate model for new ExpDesign
+ MetaModel.train_norm_design(parallel=False)
+ m_3 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+
+ # -------- Evaluate the retrained surrogate model -------
+ # Extract Modified LOO from Output
+ if pce:
+ Scores_all, varExpDesignY = [], []
+ for out_name in output_name:
+ y = MetaModel.ExpDesign.Y[out_name]
+ Scores_all.append(list(
+ MetaModel.score_dict['b_1'][out_name].values()))
+ if MetaModel.dim_red_method.lower() == 'pca':
+ pca = MetaModel.pca['b_1'][out_name]
+ components = pca.transform(y)
+ varExpDesignY.append(np.var(components,
+ axis=0))
+ else:
+ varExpDesignY.append(np.var(y, axis=0))
+ Scores = [item for sublist in Scores_all for item
+ in sublist]
+ weights = [item for sublist in varExpDesignY for item
+ in sublist]
+ ModifiedLOO = [np.average(
+ [1-score for score in Scores], weights=weights)]
+
+ print('\n')
+ print(f"Updated ModifiedLOO {util_f}:\n", ModifiedLOO)
+ print('\n')
+
+ # Compute the validation error
+ if len(MetaModel.valid_model_runs) != 0:
+ rmse, validError = self.__validError(MetaModel)
+ ValidError = list(validError.values())
+ else:
+ rmse = None
+
+ # Store updated ModifiedLOO
+ if pce:
+ SeqModifiedLOO = np.vstack(
+ (SeqModifiedLOO, ModifiedLOO))
+ if len(MetaModel.valid_model_runs) != 0:
+ SeqValidError = np.vstack(
+ (SeqValidError, ValidError))
+ m_4 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ # -------- Caclulation of BME as accuracy metric -------
+ # Check if data is provided
+ if len(obs_data) != 0:
+ # Calculate the initial BME
+ out = self.__BME_Calculator(MetaModel, obs_data,
+ TotalSigma2, rmse)
+ BME, KLD, Posterior, likes, DistHellinger = out
+ print('\n')
+ print(f"Updated BME: {BME:.2f}")
+ print(f"Updated KLD: {KLD:.2f}")
+ print('\n')
+
+ # Plot some snapshots of the posterior
+ step_snapshot = MetaModel.ExpDesign.step_snapshot
+ if post_snapshot and postcnt % step_snapshot == 0:
+ parNames = MetaModel.ExpDesign.par_names
+ print('Posterior snapshot is being plotted...')
+ self.__posteriorPlot(Posterior, parNames,
+ f'SeqPosterior_{postcnt}')
+ postcnt += 1
+ m_5 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+
+ # Check the convergence of the Mean&Std
+ if mc_ref and pce:
+ print('\n')
+ RMSE_Mean, RMSE_std = self.__error_Mean_Std()
+ print(f"Updated Mean and Std error: {RMSE_Mean:.2f}, "
+ f"{RMSE_std:.2f}")
+ print('\n')
+
+ # Store the updated BME & KLD
+ # Check if data is provided
+ if len(obs_data) != 0:
+ SeqBME = np.vstack((SeqBME, BME))
+ SeqKLD = np.vstack((SeqKLD, KLD))
+ SeqDistHellinger = np.vstack((SeqDistHellinger,
+ DistHellinger))
+ if mc_ref and pce:
+ seqRMSEMean = np.vstack((seqRMSEMean, RMSE_Mean))
+ seqRMSEStd = np.vstack((seqRMSEStd, RMSE_std))
+
+ if pce and any(LOO < mod_LOO_threshold
+ for LOO in ModifiedLOO):
+ break
+
+ print(f"Memory itr {itr_no}: I: {m_2-m_1:.2f} MB")
+ print(f"Memory itr {itr_no}: II: {m_3-m_2:.2f} MB")
+ print(f"Memory itr {itr_no}: III: {m_4-m_3:.2f} MB")
+ print(f"Memory itr {itr_no}: IV: {m_5-m_4:.2f} MB")
+ m_6 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"Memory itr {itr_no}: total: {m_6:.2f} MB")
+
+ # Clean up
+ if len(obs_data) != 0:
+ del out
+ gc.collect()
+ print()
+ print('-'*50)
+ print()
+
+ # Store updated ModifiedLOO and BME in dictonary
+ strKey = f'{util_f}_rep_{repIdx+1}'
+ if pce:
+ MetaModel.SeqModifiedLOO[strKey] = SeqModifiedLOO
+ if len(MetaModel.valid_model_runs) != 0:
+ MetaModel.seqValidError[strKey] = SeqValidError
+
+ # Check if data is provided
+ if len(obs_data) != 0:
+ MetaModel.SeqBME[strKey] = SeqBME
+ MetaModel.SeqKLD[strKey] = SeqKLD
+ if len(MetaModel.valid_likelihoods) != 0:
+ MetaModel.SeqDistHellinger[strKey] = SeqDistHellinger
+ if mc_ref and pce:
+ MetaModel.seqRMSEMean[strKey] = seqRMSEMean
+ MetaModel.seqRMSEStd[strKey] = seqRMSEStd
+
+ return MetaModel
+
+ # -------------------------------------------------------------------------
+ def util_VarBasedDesign(self, X_can, index, util_func='Entropy'):
+ """
+ Computes the exploitation scores based on:
+ active learning MacKay(ALM) and active learning Cohn (ALC)
+ Paper: Sequential Design with Mutual Information for Computer
+ Experiments (MICE): Emulation of a Tsunami Model by Beck and Guillas
+ (2016)
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ index : int
+ Model output index.
+ UtilMethod : string, optional
+ Exploitation utility function. The default is 'Entropy'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+ MetaModel = self.MetaModel
+ ED_X = MetaModel.ExpDesign.X
+ out_dict_y = MetaModel.ExpDesign.Y
+ out_names = MetaModel.ModelObj.Output.names
+
+ # Run the Metamodel for the candidate
+ X_can = X_can.reshape(1, -1)
+ Y_PC_can, std_PC_can = MetaModel.eval_metamodel(samples=X_can)
+
+ if util_func.lower() == 'alm':
+ # ----- Entropy/MMSE/active learning MacKay(ALM) -----
+ # Compute perdiction variance of the old model
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ varPCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ varPCE[KeyIdx] = np.max(canPredVar[key], axis=1)
+ score = np.max(varPCE, axis=0)
+
+ elif util_func.lower() == 'eigf':
+ # ----- Expected Improvement for Global fit -----
+ # Find closest EDX to the candidate
+ distances = distance.cdist(ED_X, X_can, 'euclidean')
+ index = np.argmin(distances)
+
+ # Compute perdiction error and variance of the old model
+ predError = {key: Y_PC_can[key] for key in out_names}
+ canPredVar = {key: std_PC_can[key]**2 for key in out_names}
+
+ # Compute perdiction error and variance of the old model
+ # Eq (5) from Liu et al.(2018)
+ EIGF_PCE = np.zeros((len(out_names), X_can.shape[0]))
+ for KeyIdx, key in enumerate(out_names):
+ residual = predError[key] - out_dict_y[key][int(index)]
+ var = canPredVar[key]
+ EIGF_PCE[KeyIdx] = np.max(residual**2 + var, axis=1)
+ score = np.max(EIGF_PCE, axis=0)
+
+ return -1 * score # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def util_BayesianActiveDesign(self, X_can, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian active design criterion (var).
+
+ It is based on the following paper:
+ Oladyshkin, Sergey, Farid Mohammadi, Ilja Kroeker, and Wolfgang Nowak.
+ "Bayesian3 active learning for the gaussian process emulator using
+ information theory." Entropy 22, no. 8 (2020): 890.
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ BAL design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_mean_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Get the data
+ obs_data = self.observations
+ n_obs = self.Model.n_obs
+ # TODO: Analytical DKL
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+
+ # priorMean, priorSigma2, Obs = np.empty((0)),np.empty((0)),np.empty((0))
+
+ # for key in list(Y_mean_can):
+ # # concatenate the measurement error
+ # Obs = np.hstack((Obs,ObservationData[key]))
+
+ # # concatenate the mean and variance of prior predictive
+ # means, stds = Y_mean_can[key][0], Y_std_can[key][0]
+ # priorMean = np.hstack((priorSigma2,means))
+ # priorSigma2 = np.hstack((priorSigma2,stds**2))
+
+ # # Covariance Matrix of prior
+ # covPrior = np.zeros((priorSigma2.shape[0], priorSigma2.shape[0]), float)
+ # np.fill_diagonal(covPrior, priorSigma2)
+
+ # # Covariance Matrix of Likelihood
+ # covLikelihood = np.zeros((sigma2Dict.shape[0], sigma2Dict.shape[0]), float)
+ # np.fill_diagonal(covLikelihood, sigma2Dict)
+
+ # # Calculate moments of the posterior (Analytical derivation)
+ # n = priorSigma2.shape[0]
+ # covPost = np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))),covLikelihood/n)
+
+ # meanPost = np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))) , Obs) + \
+ # np.dot(np.dot(covPrior,np.linalg.inv(covPrior+(covLikelihood/n))),
+ # priorMean/n)
+ # # Compute DKL from prior to posterior
+ # term1 = np.trace(np.dot(np.linalg.inv(covPrior),covPost))
+ # deltaMean = priorMean-meanPost
+ # term2 = np.dot(np.dot(deltaMean,np.linalg.inv(covPrior)),deltaMean[:,None])
+ # term3 = np.log(np.linalg.det(covPrior)/np.linalg.det(covPost))
+ # DKL = 0.5 * (term1 + term2 - n + term3)[0]
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = 20000
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Sample a distribution for a normal dist
+ # with Y_mean_can as the mean and Y_std_can as std.
+ Y_MC, std_MC = {}, {}
+ logPriorLikelihoods = np.zeros((MCsize))
+ for key in list(Y_mean_can):
+ means, stds = Y_mean_can[key][0], Y_std_can[key][0]
+ # cov = np.zeros((means.shape[0], means.shape[0]), float)
+ # np.fill_diagonal(cov, stds**2)
+
+ Y_MC[key] = np.zeros((MCsize, n_obs))
+ logsamples = np.zeros((MCsize, n_obs))
+ for i in range(n_obs):
+ NormalDensity = stats.norm(means[i], stds[i])
+ Y_MC[key][:, i] = NormalDensity.rvs(MCsize)
+ logsamples[:, i] = NormalDensity.logpdf(Y_MC[key][:, i])
+
+ logPriorLikelihoods = np.sum(logsamples, axis=1)
+ std_MC[key] = np.zeros((MCsize, means.shape[0]))
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(Y_MC, std_MC, obs_data, sigma2Dict)
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.nansum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ # Utility function Eq.2 in Ref. (2)
+ # Posterior covariance matrix after observing data y
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # TODO: Calculate the correction factor for BME
+ # BMECorrFactor = self.BME_Corr_Weight(PCE_SparseBayes_can,
+ # ObservationData, sigma2Dict)
+ # BME += BMECorrFactor
+ # Haun et al implementation
+ # U_J_d = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ U_J_d = postExpLikelihoods - logBME
+
+ # Marginal log likelihood
+ elif var == 'BME':
+ U_J_d = logBME
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ logBME = np.log(np.nanmean(likelihoods))
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # Bayesian information criterion
+ elif var == 'BIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxL = np.nanmax(likelihoods)
+ U_J_d = -2 * np.log(maxL) + np.log(n_obs) * nModelParams
+
+ # Akaike information criterion
+ elif var == 'AIC':
+ coeffs = self.MetaModel.coeffs_dict.values()
+ nModelParams = max(len(v) for val in coeffs for v in val.values())
+ maxlogL = np.log(np.nanmax(likelihoods))
+ AIC = -2 * maxlogL + 2 * nModelParams
+ # 2 * nModelParams * (nModelParams+1) / (n_obs-nModelParams-1)
+ penTerm = 0
+ U_J_d = 1*(AIC + penTerm)
+
+ # Deviance information criterion
+ elif var == 'DIC':
+ # D_theta_bar = np.mean(-2 * Likelihoods)
+ N_star_p = 0.5 * np.var(np.log(likelihoods[likelihoods != 0]))
+ Likelihoods_theta_mean = self.__normpdf(
+ Y_mean_can, Y_std_can, obs_data, sigma2Dict
+ )
+ DIC = -2 * np.log(Likelihoods_theta_mean) + 2 * N_star_p
+
+ U_J_d = DIC
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Handle inf and NaN (replace by zero)
+ if np.isnan(U_J_d) or U_J_d == -np.inf or U_J_d == np.inf:
+ U_J_d = 0.0
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+ gc.collect(generation=2)
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def update_metamodel(self, MetaModel, output, y_hat_can, univ_p_val, index,
+ new_pca=False):
+ BasisIndices = MetaModel.basis_dict[output]["y_"+str(index+1)]
+ clf_poly = MetaModel.clf_poly[output]["y_"+str(index+1)]
+ Mn = clf_poly.coef_
+ Sn = clf_poly.sigma_
+ beta = clf_poly.alpha_
+ active = clf_poly.active_
+ Psi = self.MetaModel.create_psi(BasisIndices, univ_p_val)
+
+ Sn_new_inv = np.linalg.inv(Sn)
+ Sn_new_inv += beta * np.dot(Psi[:, active].T, Psi[:, active])
+ Sn_new = np.linalg.inv(Sn_new_inv)
+
+ Mn_new = np.dot(Sn_new_inv, Mn[active]).reshape(-1, 1)
+ Mn_new += beta * np.dot(Psi[:, active].T, y_hat_can)
+ Mn_new = np.dot(Sn_new, Mn_new).flatten()
+
+ # Compute the old and new moments of PCEs
+ mean_old = Mn[0]
+ mean_new = Mn_new[0]
+ std_old = np.sqrt(np.sum(np.square(Mn[1:])))
+ std_new = np.sqrt(np.sum(np.square(Mn_new[1:])))
+
+ # Back transformation if PCA is selected.
+ if MetaModel.dim_red_method.lower() == 'pca':
+ old_pca = MetaModel.pca[output]
+ mean_old = old_pca.mean_[index]
+ mean_old += np.sum(mean_old * old_pca.components_[:, index])
+ std_old = np.sqrt(np.sum(std_old**2 *
+ old_pca.components_[:, index]**2))
+ mean_new = new_pca.mean_[index]
+ mean_new += np.sum(mean_new * new_pca.components_[:, index])
+ std_new = np.sqrt(np.sum(std_new**2 *
+ new_pca.components_[:, index]**2))
+ # print(f"mean_old: {mean_old:.2f} mean_new: {mean_new:.2f}")
+ # print(f"std_old: {std_old:.2f} std_new: {std_new:.2f}")
+ # Store the old and new moments of PCEs
+ results = {
+ 'mean_old': mean_old,
+ 'mean_new': mean_new,
+ 'std_old': std_old,
+ 'std_new': std_new
+ }
+ return results
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign_old(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ old_EDY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels using the candidate design
+ Y_PC_can, Y_std_can = self.MetaModel.eval_metamodel(
+ samples=np.array([X_can])
+ )
+
+ # Generate y from posterior predictive
+ m_size = 100
+ y_hat_samples = {}
+ for idx, key in enumerate(Model.Output.names):
+ means, stds = Y_PC_can[key][0], Y_std_can[key][0]
+ y_hat_samples[key] = np.random.multivariate_normal(
+ means, np.diag(stds), m_size)
+
+ # Create the SparseBayes-based PCE metamodel:
+ MetaModel.input_obj.poly_coeffs_flag = False
+ univ_p_val = self.MetaModel.univ_basis_vals(X_can)
+ G_n_m_all = np.zeros((m_size, len(Model.Output.names), Model.n_obs))
+
+ for i in range(m_size):
+ for idx, key in enumerate(Model.Output.names):
+ if MetaModel.dim_red_method.lower() == 'pca':
+ # Equal number of components
+ new_outputs = np.vstack(
+ (old_EDY[key], y_hat_samples[key][i])
+ )
+ new_pca, _ = MetaModel.pca_transformation(new_outputs)
+ target = new_pca.transform(
+ y_hat_samples[key][i].reshape(1, -1)
+ )[0]
+ else:
+ new_pca, target = False, y_hat_samples[key][i]
+
+ for j in range(len(target)):
+
+ # Update surrogate
+ result = self.update_metamodel(
+ MetaModel, key, target[j], univ_p_val, j, new_pca)
+
+ # Compute Expected Information Gain (Eq. 39)
+ G_n_m = np.log(result['std_old']/result['std_new']) - 1./2
+ G_n_m += result['std_new']**2 / (2*result['std_old']**2)
+ G_n_m += (result['mean_new'] - result['mean_old'])**2 /\
+ (2*result['std_old']**2)
+
+ G_n_m_all[i, idx, j] = G_n_m
+
+ U_J_d = G_n_m_all.mean(axis=(1, 2)).mean()
+ return -1 * U_J_d
+
+ # -------------------------------------------------------------------------
+ def util_BayesianDesign(self, X_can, X_MC, sigma2Dict, var='DKL'):
+ """
+ Computes scores based on Bayesian sequential design criterion (var).
+
+ Parameters
+ ----------
+ X_can : array of shape (n_samples, n_params)
+ Candidate samples.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ var : string, optional
+ Bayesian design criterion. The default is 'DKL'.
+
+ Returns
+ -------
+ float
+ Score.
+
+ """
+
+ # To avoid changes ub original aPCE object
+ Model = self.Model
+ MetaModel = deepcopy(self.MetaModel)
+ out_names = MetaModel.ModelObj.Output.names
+ if X_can.ndim == 1:
+ X_can = X_can.reshape(1, -1)
+
+ # Compute the mean and std based on the MetaModel
+ # pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+ if var == 'ALC':
+ Y_MC, Y_MC_std = MetaModel.eval_metamodel(samples=X_MC)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+ oldExpDesignY = MetaModel.ExpDesign.Y
+
+ # Evaluate the PCE metamodels at that location ???
+ Y_PC_can, Y_std_can = MetaModel.eval_metamodel(samples=X_can)
+
+ # Add all suggestion as new ExpDesign
+ NewExpDesignX = np.vstack((oldExpDesignX, X_can))
+
+ NewExpDesignY = {}
+ for key in oldExpDesignY.keys():
+ try:
+ NewExpDesignY[key] = np.vstack((oldExpDesignY[key],
+ Y_PC_can[key]))
+ except:
+ NewExpDesignY[key] = oldExpDesignY[key]
+
+ MetaModel.ExpDesign.sampling_method = 'user'
+ MetaModel.ExpDesign.X = NewExpDesignX
+ MetaModel.ExpDesign.Y = NewExpDesignY
+
+ # Train the model for the observed data using x_can
+ MetaModel.input_obj.poly_coeffs_flag = False
+ MetaModel.train_norm_design(parallel=False)
+ PCE_Model_can = MetaModel
+
+ if var.lower() == 'mi':
+ # Mutual information based on Krause et al
+ # Adapted from Beck & Guillas (MICE) paper
+ _, std_PC_can = PCE_Model_can.eval_metamodel(samples=X_can)
+ std_can = {key: std_PC_can[key] for key in out_names}
+
+ std_old = {key: Y_std_can[key] for key in out_names}
+
+ varPCE = np.zeros((len(out_names)))
+ for i, key in enumerate(out_names):
+ varPCE[i] = np.mean(std_old[key]**2/std_can[key]**2)
+ score = np.mean(varPCE)
+
+ return -1 * score
+
+ elif var.lower() == 'alc':
+ # Active learning based on Gramyc and Lee
+ # Adaptive design and analysis of supercomputer experiments Techno-
+ # metrics, 51 (2009), pp. 130–145.
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC_can, Y_MC_std_can = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Compute the score
+ score = []
+ for i, key in enumerate(out_names):
+ pce_var = Y_MC_std_can[key]**2
+ pce_var_can = Y_MC_std[key]**2
+ score.append(np.mean(pce_var-pce_var_can, axis=0))
+ score = np.mean(score)
+
+ return -1 * score
+
+ # ---------- Inner MC simulation for computing Utility Value ----------
+ # Estimation of the integral via Monte Varlo integration
+ MCsize = X_MC.shape[0]
+ ESS = 0
+
+ while ((ESS > MCsize) or (ESS < 1)):
+
+ # Enriching Monte Carlo samples if need be
+ if ESS != 0:
+ X_MC = self.MetaModel.ExpDesign.generate_samples(
+ MCsize, 'random'
+ )
+
+ # Evaluate the MetaModel at the given samples
+ Y_MC, std_MC = PCE_Model_can.eval_metamodel(samples=X_MC)
+
+ # Likelihood computation (Comparison of data and simulation
+ # results via PCE with candidate design)
+ likelihoods = self.__normpdf(
+ Y_MC, std_MC, self.observations, sigma2Dict
+ )
+
+ # Check the Effective Sample Size (1<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(likelihoods/np.sum(likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if ((ESS > MCsize) or (ESS < 1)):
+ print("--- increasing MC size---")
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (likelihoods/np.max(likelihoods)) >= unif
+
+ # -------------------- Utility functions --------------------
+ # Utility function Eq.2 in Ref. (2)
+ # Kullback-Leibler Divergence (Sergey's paper)
+ if var == 'DKL':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods, dtype=np.float128))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted]
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Haun et al implementation
+ U_J_d = np.mean(np.log(likelihoods[likelihoods != 0]) - logBME)
+
+ # U_J_d = np.sum(G_n_m_all)
+ # Ryan et al (2014) implementation
+ # importanceWeights = Likelihoods[Likelihoods!=0]/np.sum(Likelihoods[Likelihoods!=0])
+ # U_J_d = np.mean(importanceWeights*np.log(Likelihoods[Likelihoods!=0])) - logBME
+
+ # U_J_d = postExpLikelihoods - logBME
+
+ # Marginal likelihood
+ elif var == 'BME':
+
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+ U_J_d = logBME
+
+ # Bayes risk likelihood
+ elif var == 'BayesRisk':
+
+ U_J_d = -1 * np.var(likelihoods)
+
+ # Entropy-based information gain
+ elif var == 'infEntropy':
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(likelihoods))
+
+ # Posterior-based expectation of likelihoods
+ postLikelihoods = likelihoods[accepted] / np.nansum(likelihoods[accepted])
+ postExpLikelihoods = np.mean(np.log(postLikelihoods))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(logPriorLikelihoods[accepted])
+
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ U_J_d = infEntropy * -1 # -1 for minimization
+
+ # D-Posterior-precision
+ elif var == 'DPP':
+ X_Posterior = X_MC[accepted]
+ # covariance of the posterior parameters
+ U_J_d = -np.log(np.linalg.det(np.cov(X_Posterior)))
+
+ # A-Posterior-precision
+ elif var == 'APP':
+ X_Posterior = X_MC[accepted]
+ # trace of the posterior parameters
+ U_J_d = -np.log(np.trace(np.cov(X_Posterior)))
+
+ else:
+ print('The algorithm you requested has not been implemented yet!')
+
+ # Clear memory
+ del likelihoods
+ del Y_MC
+ del std_MC
+ gc.collect(generation=2)
+
+ return -1 * U_J_d # -1 is for minimization instead of maximization
+
+ # -------------------------------------------------------------------------
+ def subdomain(self, Bounds, n_new_samples):
+ """
+ Divides a domain defined by Bounds into sub domains.
+
+ Parameters
+ ----------
+ Bounds : list of tuples
+ List of lower and upper bounds.
+ n_new_samples : TYPE
+ DESCRIPTION.
+
+ Returns
+ -------
+ Subdomains : TYPE
+ DESCRIPTION.
+
+ """
+ n_params = self.MetaModel.n_params
+ n_subdomains = n_new_samples + 1
+ LinSpace = np.zeros((n_params, n_subdomains))
+
+ for i in range(n_params):
+ LinSpace[i] = np.linspace(start=Bounds[i][0], stop=Bounds[i][1],
+ num=n_subdomains)
+ Subdomains = []
+ for k in range(n_subdomains-1):
+ mylist = []
+ for i in range(n_params):
+ mylist.append((LinSpace[i, k+0], LinSpace[i, k+1]))
+ Subdomains.append(tuple(mylist))
+
+ return Subdomains
+
+ # -------------------------------------------------------------------------
+ def run_util_func(self, method, candidates, index, sigma2Dict=None,
+ var=None, X_MC=None):
+ """
+ Runs the utility function based on the given method.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ candidates : array of shape (n_samples, n_params)
+ All candidate parameter sets.
+ index : int
+ ExpDesign index.
+ sigma2Dict : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ var : string, optional
+ Utility function. The default is None.
+ X_MC : TYPE, optional
+ DESCRIPTION. The default is None.
+
+ Returns
+ -------
+ index : TYPE
+ DESCRIPTION.
+ List
+ Scores.
+
+ """
+
+ if method.lower() == 'varoptdesign':
+ # U_J_d = self.util_VarBasedDesign(candidates, index, var)
+ U_J_d = np.zeros((candidates.shape[0]))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="varoptdesign"):
+ U_J_d[idx] = self.util_VarBasedDesign(X_can, index, var)
+
+ elif method.lower() == 'bayesactdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianActiveDesign(X_can, sigma2Dict,
+ var)
+ elif method.lower() == 'bayesoptdesign':
+ NCandidate = candidates.shape[0]
+ U_J_d = np.zeros((NCandidate))
+ for idx, X_can in tqdm(enumerate(candidates), ascii=True,
+ desc="OptBayesianDesign"):
+ U_J_d[idx] = self.util_BayesianDesign(X_can, X_MC, sigma2Dict,
+ var)
+ return (index, -1 * U_J_d)
+
+ # -------------------------------------------------------------------------
+ def dual_annealing(self, method, Bounds, sigma2Dict, var, Run_No,
+ verbose=False):
+ """
+ Exploration algorithim to find the optimum parameter space.
+
+ Parameters
+ ----------
+ method : string
+ Exploitation method: `VarOptDesign`, `BayesActDesign` and
+ `BayesOptDesign`.
+ Bounds : list of tuples
+ List of lower and upper boundaries of parameters.
+ sigma2Dict : dict
+ A dictionary containing the measurement errors (sigma^2).
+ Run_No : int
+ Run number.
+ verbose : bool, optional
+ Print out a summary. The default is False.
+
+ Returns
+ -------
+ Run_No : int
+ Run number.
+ array
+ Optimial candidate.
+
+ """
+
+ Model = self.Model
+ max_func_itr = self.MetaModel.ExpDesign.max_func_itr
+
+ if method == 'VarOptDesign':
+ Res_Global = opt.dual_annealing(self.util_VarBasedDesign,
+ bounds=Bounds,
+ args=(Model, var),
+ maxfun=max_func_itr)
+
+ elif method == 'BayesOptDesign':
+ Res_Global = opt.dual_annealing(self.util_BayesianDesign,
+ bounds=Bounds,
+ args=(Model, sigma2Dict, var),
+ maxfun=max_func_itr)
+
+ if verbose:
+ print(f"global minimum: xmin = {Res_Global.x}, "
+ f"f(xmin) = {Res_Global.fun:.6f}, nfev = {Res_Global.nfev}")
+
+ return (Run_No, Res_Global.x)
+
+ # -------------------------------------------------------------------------
+ def tradoff_weights(self, tradeoff_scheme, old_EDX, old_EDY):
+ """
+ Calculates weights for exploration scores based on the requested
+ scheme: `None`, `equal`, `epsilon-decreasing` and `adaptive`.
+
+ `None`: No exploration.
+ `equal`: Same weights for exploration and exploitation scores.
+ `epsilon-decreasing`: Start with more exploration and increase the
+ influence of exploitation along the way with a exponential decay
+ function
+ `adaptive`: An adaptive method based on:
+ Liu, Haitao, Jianfei Cai, and Yew-Soon Ong. "An adaptive sampling
+ approach for Kriging metamodeling by maximizing expected prediction
+ error." Computers & Chemical Engineering 106 (2017): 171-182.
+
+ Parameters
+ ----------
+ tradeoff_scheme : string
+ Trade-off scheme for exloration and exploitation scores.
+ old_EDX : array (n_samples, n_params)
+ Old experimental design (training points).
+ old_EDY : dict
+ Old model responses (targets).
+
+ Returns
+ -------
+ exploration_weight : float
+ Exploration weight.
+ exploitation_weight: float
+ Exploitation weight.
+
+ """
+ if tradeoff_scheme is None:
+ exploration_weight = 0
+
+ elif tradeoff_scheme == 'equal':
+ exploration_weight = 0.5
+
+ elif tradeoff_scheme == 'epsilon-decreasing':
+ # epsilon-decreasing scheme
+ # Start with more exploration and increase the influence of
+ # exploitation along the way with a exponential decay function
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ tau2 = -(n_max_samples-initNSamples-1) / np.log(1e-8)
+ exploration_weight = signal.exponential(n_max_samples-initNSamples,
+ 0, tau2, False)[itrNumber]
+
+ elif tradeoff_scheme == 'adaptive':
+
+ # Extract itrNumber
+ initNSamples = self.MetaModel.ExpDesign.n_init_samples
+ n_max_samples = self.MetaModel.ExpDesign.n_max_samples
+ itrNumber = (self.MetaModel.ExpDesign.X.shape[0] - initNSamples)
+ itrNumber //= self.MetaModel.ExpDesign.n_new_samples
+
+ if itrNumber == 0:
+ exploration_weight = 0.5
+ else:
+ # New adaptive trade-off according to Liu et al. (2017)
+ # Mean squared error for last design point
+ last_EDX = old_EDX[-1].reshape(1, -1)
+ lastPCEY, _ = self.MetaModel.eval_metamodel(samples=last_EDX)
+ pce_y = np.array(list(lastPCEY.values()))[:, 0]
+ y = np.array(list(old_EDY.values()))[:, -1, :]
+ mseError = mean_squared_error(pce_y, y)
+
+ # Mean squared CV - error for last design point
+ pce_y_prev = np.array(list(self._y_hat_prev.values()))[:, 0]
+ mseCVError = mean_squared_error(pce_y_prev, y)
+
+ exploration_weight = min([0.5*mseError/mseCVError, 1])
+
+ # Exploitation weight
+ exploitation_weight = 1 - exploration_weight
+
+ return exploration_weight, exploitation_weight
+
+ # -------------------------------------------------------------------------
+ def opt_SeqDesign(self, sigma2, n_candidates=5, var='DKL'):
+ """
+ Runs optimal sequential design.
+
+ Parameters
+ ----------
+ sigma2 : dict, optional
+ A dictionary containing the measurement errors (sigma^2). The
+ default is None.
+ n_candidates : int, optional
+ Number of candidate samples. The default is 5.
+ var : string, optional
+ Utility function. The default is None.
+
+ Raises
+ ------
+ NameError
+ Wrong utility function.
+
+ Returns
+ -------
+ Xnew : array (n_samples, n_params)
+ Selected new training point(s).
+ """
+
+ # Initialization
+ MetaModel = self.MetaModel
+ Bounds = MetaModel.bound_tuples
+ n_new_samples = MetaModel.ExpDesign.n_new_samples
+ explore_method = MetaModel.ExpDesign.explore_method
+ exploit_method = MetaModel.ExpDesign.exploit_method
+ n_cand_groups = MetaModel.ExpDesign.n_cand_groups
+ tradeoff_scheme = MetaModel.ExpDesign.tradeoff_scheme
+
+ old_EDX = MetaModel.ExpDesign.X
+ old_EDY = MetaModel.ExpDesign.Y.copy()
+ ndim = MetaModel.ExpDesign.X.shape[1]
+ OutputNames = MetaModel.ModelObj.Output.names
+
+ # -----------------------------------------
+ # ----------- CUSTOMIZED METHODS ----------
+ # -----------------------------------------
+ # Utility function exploit_method provided by user
+ if exploit_method.lower() == 'user':
+
+ Xnew, filteredSamples = MetaModel.ExpDesign.ExploitFunction(self)
+
+ print("\n")
+ print("\nXnew:\n", Xnew)
+
+ return Xnew, filteredSamples
+
+ # -----------------------------------------
+ # ---------- EXPLORATION METHODS ----------
+ # -----------------------------------------
+ if explore_method == 'dual annealing':
+ # ------- EXPLORATION: OPTIMIZATION -------
+ import time
+ start_time = time.time()
+
+ # Divide the domain to subdomains
+ args = []
+ subdomains = self.subdomain(Bounds, n_new_samples)
+ for i in range(n_new_samples):
+ args.append((exploit_method, subdomains[i], sigma2, var, i))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.dual_annealing, args).get()
+
+ # Close the pool
+ pool.close()
+
+ Xnew = np.array([results[i][1] for i in range(n_new_samples)])
+
+ print("\nXnew:\n", Xnew)
+
+ elapsed_time = time.time() - start_time
+ print("\n")
+ print(f"elapsed_time: {round(elapsed_time,2)} sec.")
+ print('-'*20)
+
+ elif explore_method == 'LOOCV':
+ # -----------------------------------------------------------------
+ # TODO: LOOCV model construnction based on Feng et al. (2020)
+ # 'LOOCV':
+ # Initilize the ExploitScore array
+
+ # Generate random samples
+ allCandidates = MetaModel.ExpDesign.generate_samples(n_candidates,
+ 'random')
+
+ # Construct error model based on LCerror
+ errorModel = MetaModel.create_ModelError(old_EDX, self.LCerror)
+ self.errorModel.append(copy(errorModel))
+
+ # Evaluate the error models for allCandidates
+ eLCAllCands, _ = errorModel.eval_errormodel(allCandidates)
+ # Select the maximum as the representative error
+ eLCAllCands = np.dstack(eLCAllCands.values())
+ eLCAllCandidates = np.max(eLCAllCands, axis=1)[:, 0]
+
+ # Normalize the error w.r.t the maximum error
+ scoreExploration = eLCAllCandidates / np.sum(eLCAllCandidates)
+
+ else:
+ # ------- EXPLORATION: SPACE-FILLING DESIGN -------
+ # Generate candidate samples from Exploration class
+ explore = Exploration(MetaModel, n_candidates)
+ explore.w = 100 # * ndim #500
+ # Select criterion (mc-intersite-proj-th, mc-intersite-proj)
+ explore.mc_criterion = 'mc-intersite-proj'
+ allCandidates, scoreExploration = explore.get_exploration_samples()
+
+ # Temp: ---- Plot all candidates -----
+ if ndim == 2:
+ def plotter(points, allCandidates, Method,
+ scoreExploration=None):
+ if Method == 'Voronoi':
+ from scipy.spatial import Voronoi, voronoi_plot_2d
+ vor = Voronoi(points)
+ fig = voronoi_plot_2d(vor)
+ ax1 = fig.axes[0]
+ else:
+ fig = plt.figure()
+ ax1 = fig.add_subplot(111)
+ ax1.scatter(points[:, 0], points[:, 1], s=10, c='r',
+ marker="s", label='Old Design Points')
+ ax1.scatter(allCandidates[:, 0], allCandidates[:, 1], s=10,
+ c='b', marker="o", label='Design candidates')
+ for i in range(points.shape[0]):
+ txt = 'p'+str(i+1)
+ ax1.annotate(txt, (points[i, 0], points[i, 1]))
+ if scoreExploration is not None:
+ for i in range(allCandidates.shape[0]):
+ txt = str(round(scoreExploration[i], 5))
+ ax1.annotate(txt, (allCandidates[i, 0],
+ allCandidates[i, 1]))
+
+ plt.xlim(self.bound_tuples[0])
+ plt.ylim(self.bound_tuples[1])
+ # plt.show()
+ plt.legend(loc='upper left')
+
+ # -----------------------------------------
+ # --------- EXPLOITATION METHODS ----------
+ # -----------------------------------------
+ if exploit_method == 'BayesOptDesign' or\
+ exploit_method == 'BayesActDesign':
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\n Exploration weight={explore_w:0.3f} "
+ f"Exploitation weight={exploit_w:0.3f}\n")
+
+ # ------- EXPLOITATION: BayesOptDesign & ActiveLearning -------
+ if explore_w != 1.0:
+
+ # Create a sample pool for rejection sampling
+ MCsize = 15000
+ X_MC = MetaModel.ExpDesign.generate_samples(MCsize, 'random')
+ candidates = MetaModel.ExpDesign.generate_samples(
+ MetaModel.ExpDesign.max_func_itr, 'latin_hypercube')
+
+ # Split the candidates in groups for multiprocessing
+ split_cand = np.array_split(
+ candidates, n_cand_groups, axis=0
+ )
+
+ results = Parallel(n_jobs=-1, backend='threading')(
+ delayed(self.run_util_func)(
+ exploit_method, split_cand[i], i, sigma2, var, X_MC)
+ for i in range(n_cand_groups))
+ # out = map(self.run_util_func,
+ # [exploit_method]*n_cand_groups,
+ # split_cand,
+ # range(n_cand_groups),
+ # [sigma2] * n_cand_groups,
+ # [var] * n_cand_groups,
+ # [X_MC] * n_cand_groups
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ U_J_d = np.concatenate([results[NofE][1] for NofE in
+ range(n_cand_groups)])
+
+ # Check if all scores are inf
+ if np.isinf(U_J_d).all() or np.isnan(U_J_d).all():
+ U_J_d = np.ones(len(U_J_d))
+
+ # Get the expected value (mean) of the Utility score
+ # for each cell
+ if explore_method == 'Voronoi':
+ U_J_d = np.mean(U_J_d.reshape(-1, n_candidates), axis=1)
+
+ # create surrogate model for U_J_d
+ from sklearn.preprocessing import MinMaxScaler
+ # Take care of inf entries
+ good_indices = [i for i, arr in enumerate(U_J_d)
+ if np.isfinite(arr).all()]
+ scaler = MinMaxScaler()
+ X_S = scaler.fit_transform(candidates[good_indices])
+ gp = MetaModel.gaussian_process_emulator(
+ X_S, U_J_d[good_indices], autoSelect=True
+ )
+ U_J_d = gp.predict(scaler.transform(allCandidates))
+
+ # Normalize U_J_d
+ norm_U_J_d = U_J_d / np.sum(U_J_d)
+ print("norm_U_J_d:\n", norm_U_J_d)
+ else:
+ norm_U_J_d = np.zeros((len(scoreExploration)))
+
+ # ------- Calculate Total score -------
+ # ------- Trade off between EXPLORATION & EXPLOITATION -------
+ # Total score
+ totalScore = exploit_w * norm_U_J_d
+ totalScore += explore_w * scoreExploration
+
+ # temp: Plot
+ # dim = self.ExpDesign.X.shape[1]
+ # if dim == 2:
+ # plotter(self.ExpDesign.X, allCandidates, explore_method)
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ # select the requested number of samples
+ if explore_method == 'Voronoi':
+ Xnew = np.zeros((n_new_samples, ndim))
+ for i, idx in enumerate(bestIdx):
+ X_can = explore.closestPoints[idx]
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+ else:
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ elif exploit_method == 'VarOptDesign':
+ # ------- EXPLOITATION: VarOptDesign -------
+ UtilMethod = var
+
+ # ------- Calculate Exoploration weight -------
+ # Compute exploration weight based on trade off scheme
+ explore_w, exploit_w = self.tradoff_weights(tradeoff_scheme,
+ old_EDX,
+ old_EDY)
+ print(f"\nweightExploration={explore_w:0.3f} "
+ f"weightExploitation={exploit_w:0.3f}")
+
+ # Generate candidate samples from Exploration class
+ nMeasurement = old_EDY[OutputNames[0]].shape[1]
+
+ # Find sensitive region
+ if UtilMethod == 'LOOCV':
+ LCerror = MetaModel.LCerror
+ allModifiedLOO = np.zeros((len(old_EDX), len(OutputNames),
+ nMeasurement))
+ for y_idx, y_key in enumerate(OutputNames):
+ for idx, key in enumerate(LCerror[y_key].keys()):
+ allModifiedLOO[:, y_idx, idx] = abs(
+ LCerror[y_key][key])
+
+ ExploitScore = np.max(np.max(allModifiedLOO, axis=1), axis=1)
+
+ elif UtilMethod in ['EIGF', 'ALM']:
+ # ----- All other in ['EIGF', 'ALM'] -----
+ # Initilize the ExploitScore array
+ ExploitScore = np.zeros((len(old_EDX), len(OutputNames)))
+
+ # Split the candidates in groups for multiprocessing
+ if explore_method != 'Voronoi':
+ split_cand = np.array_split(allCandidates,
+ n_cand_groups,
+ axis=0)
+ goodSampleIdx = range(n_cand_groups)
+ else:
+ # Find indices of the Vornoi cells with samples
+ goodSampleIdx = []
+ for idx in range(len(explore.closest_points)):
+ if len(explore.closest_points[idx]) != 0:
+ goodSampleIdx.append(idx)
+ split_cand = explore.closest_points
+
+ # Split the candidates in groups for multiprocessing
+ args = []
+ for index in goodSampleIdx:
+ args.append((exploit_method, split_cand[index], index,
+ sigma2, var))
+
+ # Multiprocessing
+ pool = multiprocessing.Pool(multiprocessing.cpu_count())
+ # With Pool.starmap_async()
+ results = pool.starmap_async(self.run_util_func, args).get()
+
+ # Close the pool
+ pool.close()
+ # out = map(self.run_util_func,
+ # [exploit_method]*len(goodSampleIdx),
+ # split_cand,
+ # range(len(goodSampleIdx)),
+ # [sigma2] * len(goodSampleIdx),
+ # [var] * len(goodSampleIdx)
+ # )
+ # results = list(out)
+
+ # Retrieve the results and append them
+ if explore_method == 'Voronoi':
+ ExploitScore = [np.mean(results[k][1]) for k in
+ range(len(goodSampleIdx))]
+ else:
+ ExploitScore = np.concatenate(
+ [results[k][1] for k in range(len(goodSampleIdx))])
+
+ else:
+ raise NameError('The requested utility function is not '
+ 'available.')
+
+ # print("ExploitScore:\n", ExploitScore)
+
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ # Total score
+ # Normalize U_J_d
+ ExploitScore = ExploitScore / np.sum(ExploitScore)
+ totalScore = exploit_w * ExploitScore
+ totalScore += explore_w * scoreExploration
+
+ temp = totalScore.copy()
+ sorted_idxtotalScore = np.argsort(temp, axis=0)[::-1]
+ bestIdx = sorted_idxtotalScore[:n_new_samples]
+
+ Xnew = np.zeros((n_new_samples, ndim))
+ if explore_method != 'Voronoi':
+ Xnew = allCandidates[bestIdx]
+ else:
+ for i, idx in enumerate(bestIdx.flatten()):
+ X_can = explore.closest_points[idx]
+ # plotter(self.ExpDesign.X, X_can, explore_method,
+ # scoreExploration=None)
+
+ # Calculate the maxmin score for the region of interest
+ newSamples, maxminScore = explore.get_mc_samples(X_can)
+
+ # select the requested number of samples
+ Xnew[i] = newSamples[np.argmax(maxminScore)]
+
+ elif exploit_method == 'alphabetic':
+ # ------- EXPLOITATION: ALPHABETIC -------
+ Xnew = self.util_AlphOptDesign(allCandidates, var)
+
+ elif exploit_method == 'Space-filling':
+ # ------- EXPLOITATION: SPACE-FILLING -------
+ totalScore = scoreExploration
+
+ # ------- Select the best candidate -------
+ # find an optimal point subset to add to the initial design by
+ # maximization of the utility score and taking care of NaN values
+ temp = totalScore.copy()
+ temp[np.isnan(totalScore)] = -np.inf
+ sorted_idxtotalScore = np.argsort(temp)[::-1]
+
+ # select the requested number of samples
+ Xnew = allCandidates[sorted_idxtotalScore[:n_new_samples]]
+
+ else:
+ raise NameError('The requested design method is not available.')
+
+ print("\n")
+ print("\nRun No. {}:".format(old_EDX.shape[0]+1))
+ print("Xnew:\n", Xnew)
+ gc.collect()
+
+ return Xnew, None
+
+ # -------------------------------------------------------------------------
+ def util_AlphOptDesign(self, candidates, var='D-Opt'):
+ """
+ Enriches the Experimental design with the requested alphabetic
+ criterion based on exploring the space with number of sampling points.
+
+ Ref: Hadigol, M., & Doostan, A. (2018). Least squares polynomial chaos
+ expansion: A review of sampling strategies., Computer Methods in
+ Applied Mechanics and Engineering, 332, 382-407.
+
+ Arguments
+ ---------
+ NCandidate : int
+ Number of candidate points to be searched
+
+ var : string
+ Alphabetic optimality criterion
+
+ Returns
+ -------
+ X_new : array of shape (1, n_params)
+ The new sampling location in the input space.
+ """
+ MetaModelOrig = self
+ Model = self.Model
+ n_new_samples = MetaModelOrig.ExpDesign.n_new_samples
+ NCandidate = candidates.shape[0]
+
+ # TODO: Loop over outputs
+ OutputName = Model.Output.names[0]
+
+ # To avoid changes ub original aPCE object
+ MetaModel = deepcopy(MetaModelOrig)
+
+ # Old Experimental design
+ oldExpDesignX = MetaModel.ExpDesign.X
+
+ # TODO: Only one psi can be selected.
+ # Suggestion: Go for the one with the highest LOO error
+ Scores = list(MetaModel.score_dict[OutputName].values())
+ ModifiedLOO = [1-score for score in Scores]
+ outIdx = np.argmax(ModifiedLOO)
+
+ # Initialize Phi to save the criterion's values
+ Phi = np.zeros((NCandidate))
+
+ BasisIndices = MetaModelOrig.basis_dict[OutputName]["y_"+str(outIdx+1)]
+ P = len(BasisIndices)
+
+ # ------ Old Psi ------------
+ univ_p_val = MetaModelOrig.univ_basis_vals(oldExpDesignX)
+ Psi = MetaModelOrig.create_psi(BasisIndices, univ_p_val)
+
+ # ------ New candidates (Psi_c) ------------
+ # Assemble Psi_c
+ univ_p_val_c = self.univ_basis_vals(candidates)
+ Psi_c = self.create_psi(BasisIndices, univ_p_val_c)
+
+ for idx in range(NCandidate):
+
+ # Include the new row to the original Psi
+ Psi_cand = np.vstack((Psi, Psi_c[idx]))
+
+ # Information matrix
+ PsiTPsi = np.dot(Psi_cand.T, Psi_cand)
+ M = PsiTPsi / (len(oldExpDesignX)+1)
+
+ if np.linalg.cond(PsiTPsi) > 1e-12 \
+ and np.linalg.cond(PsiTPsi) < 1 / sys.float_info.epsilon:
+ # faster
+ invM = linalg.solve(M, sparse.eye(PsiTPsi.shape[0]).toarray())
+ else:
+ # stabler
+ invM = np.linalg.pinv(M)
+
+ # ---------- Calculate optimality criterion ----------
+ # Optimality criteria according to Section 4.5.1 in Ref.
+
+ # D-Opt
+ if var == 'D-Opt':
+ Phi[idx] = (np.linalg.det(invM)) ** (1/P)
+
+ # A-Opt
+ elif var == 'A-Opt':
+ Phi[idx] = np.trace(invM)
+
+ # K-Opt
+ elif var == 'K-Opt':
+ Phi[idx] = np.linalg.cond(M)
+
+ else:
+ raise Exception('The optimality criterion you requested has '
+ 'not been implemented yet!')
+
+ # find an optimal point subset to add to the initial design
+ # by minimization of the Phi
+ sorted_idxtotalScore = np.argsort(Phi)
+
+ # select the requested number of samples
+ Xnew = candidates[sorted_idxtotalScore[:n_new_samples]]
+
+ return Xnew
+
+ # -------------------------------------------------------------------------
+ def __normpdf(self, y_hat_pce, std_pce, obs_data, total_sigma2s,
+ rmse=None):
+
+ Model = self.Model
+ likelihoods = 1.0
+
+ # Loop over the outputs
+ for idx, out in enumerate(Model.Output.names):
+
+ # (Meta)Model Output
+ nsamples, nout = y_hat_pce[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout].values
+
+ # Surrogate error if valid dataset is given.
+ if rmse is not None:
+ tot_sigma2s += rmse[out]**2
+
+ likelihoods *= stats.multivariate_normal.pdf(
+ y_hat_pce[out], data, np.diag(tot_sigma2s),
+ allow_singular=True)
+ self.Likelihoods = likelihoods
+
+ return likelihoods
+
+ # -------------------------------------------------------------------------
+ def __corr_factor_BME(self, obs_data, total_sigma2s, logBME):
+ """
+ Calculates the correction factor for BMEs.
+ """
+ MetaModel = self.MetaModel
+ samples = MetaModel.ExpDesign.X # valid_samples
+ model_outputs = MetaModel.ExpDesign.Y # valid_model_runs
+ Model = MetaModel.ModelObj
+ n_samples = samples.shape[0]
+
+ # Extract the requested model outputs for likelihood calulation
+ output_names = Model.Output.names
+
+ # TODO: Evaluate MetaModel on the experimental design and ValidSet
+ OutputRS, stdOutputRS = MetaModel.eval_metamodel(samples=samples)
+
+ logLik_data = np.zeros((n_samples))
+ logLik_model = np.zeros((n_samples))
+ # Loop over the outputs
+ for idx, out in enumerate(output_names):
+
+ # (Meta)Model Output
+ nsamples, nout = model_outputs[out].shape
+
+ # Prepare data and remove NaN
+ try:
+ data = obs_data[out].values[~np.isnan(obs_data[out])]
+ except AttributeError:
+ data = obs_data[out][~np.isnan(obs_data[out])]
+
+ # Prepare sigma2s
+ non_nan_indices = ~np.isnan(total_sigma2s[out])
+ tot_sigma2s = total_sigma2s[out][non_nan_indices][:nout]
+
+ # Covariance Matrix
+ covMatrix_data = np.diag(tot_sigma2s)
+
+ for i, sample in enumerate(samples):
+
+ # Simulation run
+ y_m = model_outputs[out][i]
+
+ # Surrogate prediction
+ y_m_hat = OutputRS[out][i]
+
+ # CovMatrix with the surrogate error
+ # covMatrix = np.diag(stdOutputRS[out][i]**2)
+ covMatrix = np.diag((y_m-y_m_hat)**2)
+ covMatrix = np.diag(
+ np.mean((model_outputs[out]-OutputRS[out]), axis=0)**2
+ )
+
+ # Compute likelilhood output vs data
+ logLik_data[i] += self.__logpdf(
+ y_m_hat, data, covMatrix_data
+ )
+
+ # Compute likelilhood output vs surrogate
+ logLik_model[i] += self.__logpdf(y_m_hat, y_m, covMatrix)
+
+ # Weight
+ logLik_data -= logBME
+ weights = np.exp(logLik_model+logLik_data)
+
+ return np.log(np.mean(weights))
+
+ # -------------------------------------------------------------------------
+ def __logpdf(self, x, mean, cov):
+ """
+ computes the likelihood based on a multivariate normal distribution.
+
+ Parameters
+ ----------
+ x : TYPE
+ DESCRIPTION.
+ mean : array_like
+ Observation data.
+ cov : 2d array
+ Covariance matrix of the distribution.
+
+ Returns
+ -------
+ log_lik : float
+ Log likelihood.
+
+ """
+ n = len(mean)
+ L = linalg.cholesky(cov, lower=True)
+ beta = np.sum(np.log(np.diag(L)))
+ dev = x - mean
+ alpha = dev.dot(linalg.cho_solve((L, True), dev))
+ log_lik = -0.5 * alpha - beta - n / 2. * np.log(2 * np.pi)
+
+ return log_lik
+
+ # -------------------------------------------------------------------------
+ def __posteriorPlot(self, posterior, par_names, key):
+
+ # Initialization
+ newpath = (r'Outputs_SeqPosteriorComparison/posterior')
+ os.makedirs(newpath, exist_ok=True)
+
+ bound_tuples = self.MetaModel.bound_tuples
+ n_params = len(par_names)
+ font_size = 40
+ if n_params == 2:
+
+ figPosterior, ax = plt.subplots(figsize=(15, 15))
+
+ sns.kdeplot(x=posterior[:, 0], y=posterior[:, 1],
+ fill=True, ax=ax, cmap=plt.cm.jet,
+ clip=bound_tuples)
+ # Axis labels
+ plt.xlabel(par_names[0], fontsize=font_size)
+ plt.ylabel(par_names[1], fontsize=font_size)
+
+ # Set axis limit
+ plt.xlim(bound_tuples[0])
+ plt.ylim(bound_tuples[1])
+
+ # Increase font size
+ plt.xticks(fontsize=font_size)
+ plt.yticks(fontsize=font_size)
+
+ # Switch off the grids
+ plt.grid(False)
+
+ else:
+ import corner
+ figPosterior = corner.corner(posterior, labels=par_names,
+ title_fmt='.2e', show_titles=True,
+ title_kwargs={"fontsize": 12})
+
+ figPosterior.savefig(f'./{newpath}/{key}.pdf', bbox_inches='tight')
+ plt.close()
+
+ # Save the posterior as .npy
+ np.save(f'./{newpath}/{key}.npy', posterior)
+
+ return figPosterior
+
+ # -------------------------------------------------------------------------
+ def __hellinger_distance(self, P, Q):
+ """
+ Hellinger distance between two continuous distributions.
+
+ The maximum distance 1 is achieved when P assigns probability zero to
+ every set to which Q assigns a positive probability, and vice versa.
+ 0 (identical) and 1 (maximally different)
+
+ Parameters
+ ----------
+ P : array
+ Reference likelihood.
+ Q : array
+ Estimated likelihood.
+
+ Returns
+ -------
+ float
+ Hellinger distance of two distributions.
+
+ """
+ mu1 = P.mean()
+ Sigma1 = np.std(P)
+
+ mu2 = Q.mean()
+ Sigma2 = np.std(Q)
+
+ term1 = np.sqrt(2*Sigma1*Sigma2 / (Sigma1**2 + Sigma2**2))
+
+ term2 = np.exp(-.25 * (mu1 - mu2)**2 / (Sigma1**2 + Sigma2**2))
+
+ H_squared = 1 - term1 * term2
+
+ return np.sqrt(H_squared)
+
+ # -------------------------------------------------------------------------
+ def __BME_Calculator(self, MetaModel, obs_data, sigma2Dict, rmse=None):
+ """
+ This function computes the Bayesian model evidence (BME) via Monte
+ Carlo integration.
+
+ """
+ # Initializations
+ valid_likelihoods = MetaModel.valid_likelihoods
+
+ post_snapshot = MetaModel.ExpDesign.post_snapshot
+ if post_snapshot or len(valid_likelihoods) != 0:
+ newpath = (r'Outputs_SeqPosteriorComparison/likelihood_vs_ref')
+ os.makedirs(newpath, exist_ok=True)
+
+ SamplingMethod = 'random'
+ MCsize = 10000
+ ESS = 0
+
+ # Estimation of the integral via Monte Varlo integration
+ while (ESS > MCsize) or (ESS < 1):
+
+ # Generate samples for Monte Carlo simulation
+ X_MC = MetaModel.ExpDesign.generate_samples(
+ MCsize, SamplingMethod
+ )
+
+ # Monte Carlo simulation for the candidate design
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ Y_MC, std_MC = MetaModel.eval_metamodel(samples=X_MC)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"\nMemory eval_metamodel in BME: {m_2-m_1:.2f} MB")
+
+ # Likelihood computation (Comparison of data and
+ # simulation results via PCE with candidate design)
+ Likelihoods = self.__normpdf(
+ Y_MC, std_MC, obs_data, sigma2Dict, rmse
+ )
+
+ # Check the Effective Sample Size (1000<ESS<MCsize)
+ ESS = 1 / np.sum(np.square(Likelihoods/np.sum(Likelihoods)))
+
+ # Enlarge sample size if it doesn't fulfill the criteria
+ if (ESS > MCsize) or (ESS < 1):
+ print(f'ESS={ESS} MC size should be larger.')
+ MCsize *= 10
+ ESS = 0
+
+ # Rejection Step
+ # Random numbers between 0 and 1
+ unif = np.random.rand(1, MCsize)[0]
+
+ # Reject the poorly performed prior
+ accepted = (Likelihoods/np.max(Likelihoods)) >= unif
+ X_Posterior = X_MC[accepted]
+
+ # ------------------------------------------------------------
+ # --- Kullback-Leibler Divergence & Information Entropy ------
+ # ------------------------------------------------------------
+ # Prior-based estimation of BME
+ logBME = np.log(np.nanmean(Likelihoods))
+
+ # TODO: Correction factor
+ # log_weight = self.__corr_factor_BME(obs_data, sigma2Dict, logBME)
+
+ # Posterior-based expectation of likelihoods
+ postExpLikelihoods = np.mean(np.log(Likelihoods[accepted]))
+
+ # Posterior-based expectation of prior densities
+ postExpPrior = np.mean(
+ np.log(MetaModel.ExpDesign.JDist.pdf(X_Posterior.T))
+ )
+
+ # Calculate Kullback-Leibler Divergence
+ # KLD = np.mean(np.log(Likelihoods[Likelihoods!=0])- logBME)
+ KLD = postExpLikelihoods - logBME
+
+ # Information Entropy based on Entropy paper Eq. 38
+ infEntropy = logBME - postExpPrior - postExpLikelihoods
+
+ # If post_snapshot is True, plot likelihood vs refrence
+ if post_snapshot or len(valid_likelihoods) != 0:
+ # Hellinger distance
+ ref_like = np.log(valid_likelihoods[valid_likelihoods > 0])
+ est_like = np.log(Likelihoods[Likelihoods > 0])
+ distHellinger = self.__hellinger_distance(ref_like, est_like)
+
+ idx = len([name for name in os.listdir(newpath) if 'Likelihoods_'
+ in name and os.path.isfile(os.path.join(newpath, name))])
+ fig, ax = plt.subplots()
+ try:
+ sns.kdeplot(np.log(valid_likelihoods[valid_likelihoods > 0]),
+ shade=True, color="g", label='Ref. Likelihood')
+ sns.kdeplot(np.log(Likelihoods[Likelihoods > 0]), shade=True,
+ color="b", label='Likelihood with PCE')
+ except:
+ pass
+
+ text = f"Hellinger Dist.={distHellinger:.3f}\n logBME={logBME:.3f}"
+ "\n DKL={KLD:.3f}"
+
+ plt.text(0.05, 0.75, text, bbox=dict(facecolor='wheat',
+ edgecolor='black',
+ boxstyle='round,pad=1'),
+ transform=ax.transAxes)
+
+ fig.savefig(f'./{newpath}/Likelihoods_{idx}.pdf',
+ bbox_inches='tight')
+ plt.close()
+
+ else:
+ distHellinger = 0.0
+
+ # Bayesian inference with Emulator only for 2D problem
+ if post_snapshot and MetaModel.n_params == 2 and not idx % 5:
+ from bayes_inference.bayes_inference import BayesInference
+ from bayes_inference.discrepancy import Discrepancy
+ import pandas as pd
+ BayesOpts = BayesInference(MetaModel)
+ BayesOpts.emulator = True
+ BayesOpts.plot_post_pred = False
+
+ # Select the inference method
+ import emcee
+ BayesOpts.inference_method = "MCMC"
+ # Set the MCMC parameters passed to self.mcmc_params
+ BayesOpts.mcmc_params = {
+ 'n_steps': 1e5,
+ 'n_walkers': 30,
+ 'moves': emcee.moves.KDEMove(),
+ 'verbose': False
+ }
+
+ # ----- Define the discrepancy model -------
+ obs_data = pd.DataFrame(obs_data, columns=self.Model.Output.names)
+ BayesOpts.measurement_error = obs_data
+
+ # # -- (Option B) --
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = obs_data**2
+ BayesOpts.Discrepancy = DiscrepancyOpts
+ # Start the calibration/inference
+ Bayes_PCE = BayesOpts.create_inference()
+ X_Posterior = Bayes_PCE.posterior_df.values
+
+ # Clean up
+ del Y_MC, std_MC
+ gc.collect()
+
+ return (logBME, KLD, X_Posterior, Likelihoods, distHellinger)
+
+ # -------------------------------------------------------------------------
+ def __validError(self, MetaModel):
+
+ # MetaModel = self.MetaModel
+ Model = MetaModel.ModelObj
+ OutputName = Model.Output.names
+
+ # Extract the original model with the generated samples
+ valid_samples = MetaModel.valid_samples
+ valid_model_runs = MetaModel.valid_model_runs
+
+ # Run the PCE model with the generated samples
+ m_1 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ valid_PCE_runs, valid_PCE_std = MetaModel.eval_metamodel(samples=valid_samples)
+ m_2 = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024
+ print(f"\nMemory eval_metamodel: {m_2-m_1:.2f} MB")
+
+ rms_error = {}
+ valid_error = {}
+ # Loop over the keys and compute RMSE error.
+ for key in OutputName:
+ rms_error[key] = mean_squared_error(
+ valid_model_runs[key], valid_PCE_runs[key],
+ multioutput='raw_values',
+ sample_weight=None,
+ squared=False)
+
+ # Validation error
+ valid_error[key] = (rms_error[key]**2)
+ valid_error[key] /= np.var(valid_model_runs[key], ddof=1, axis=0)
+
+ # Print a report table
+ print("\n>>>>> Updated Errors of {} <<<<<".format(key))
+ print("\nIndex | RMSE | Validation Error")
+ print('-'*35)
+ print('\n'.join(f'{i+1} | {k:.3e} | {j:.3e}' for i, (k, j)
+ in enumerate(zip(rms_error[key],
+ valid_error[key]))))
+
+ return rms_error, valid_error
+
+ # -------------------------------------------------------------------------
+ def __error_Mean_Std(self):
+
+ MetaModel = self.MetaModel
+ # Extract the mean and std provided by user
+ df_MCReference = MetaModel.ModelObj.mc_reference
+
+ # Compute the mean and std based on the MetaModel
+ pce_means, pce_stds = self._compute_pce_moments(MetaModel)
+
+ # Compute the root mean squared error
+ for output in MetaModel.ModelObj.Output.names:
+
+ # Compute the error between mean and std of MetaModel and OrigModel
+ RMSE_Mean = mean_squared_error(
+ df_MCReference['mean'], pce_means[output], squared=False
+ )
+ RMSE_std = mean_squared_error(
+ df_MCReference['std'], pce_means[output], squared=False
+ )
+
+ return RMSE_Mean, RMSE_std
+
+ # -------------------------------------------------------------------------
+ def _compute_pce_moments(self, MetaModel):
+ """
+ Computes the first two moments using the PCE-based meta-model.
+
+ Returns
+ -------
+ pce_means: dict
+ The first moment (mean) of the surrogate.
+ pce_stds: dict
+ The second moment (standard deviation) of the surrogate.
+
+ """
+ outputs = MetaModel.ModelObj.Output.names
+ pce_means_b = {}
+ pce_stds_b = {}
+
+ # Loop over bootstrap iterations
+ for b_i in range(MetaModel.n_bootstrap_itrs):
+ # Loop over the metamodels
+ coeffs_dicts = MetaModel.coeffs_dict[f'b_{b_i+1}'].items()
+ means = {}
+ stds = {}
+ for output, coef_dict in coeffs_dicts:
+
+ pce_mean = np.zeros((len(coef_dict)))
+ pce_var = np.zeros((len(coef_dict)))
+
+ for index, values in coef_dict.items():
+ idx = int(index.split('_')[1]) - 1
+ coeffs = MetaModel.coeffs_dict[f'b_{b_i+1}'][output][index]
+
+ # Mean = c_0
+ if coeffs[0] != 0:
+ pce_mean[idx] = coeffs[0]
+ else:
+ clf_poly = MetaModel.clf_poly[f'b_{b_i+1}'][output]
+ pce_mean[idx] = clf_poly[index].intercept_
+ # Var = sum(coeffs[1:]**2)
+ pce_var[idx] = np.sum(np.square(coeffs[1:]))
+
+ # Save predictions for each output
+ if MetaModel.dim_red_method.lower() == 'pca':
+ PCA = MetaModel.pca[f'b_{b_i+1}'][output]
+ means[output] = PCA.mean_ + np.dot(
+ pce_mean, PCA.components_)
+ stds[output] = np.sqrt(np.dot(pce_var,
+ PCA.components_**2))
+ else:
+ means[output] = pce_mean
+ stds[output] = np.sqrt(pce_var)
+
+ # Save predictions for each bootstrap iteration
+ pce_means_b[b_i] = means
+ pce_stds_b[b_i] = stds
+
+ # Change the order of nesting
+ mean_all = {}
+ for i in sorted(pce_means_b):
+ for k, v in pce_means_b[i].items():
+ if k not in mean_all:
+ mean_all[k] = [None] * len(pce_means_b)
+ mean_all[k][i] = v
+ std_all = {}
+ for i in sorted(pce_stds_b):
+ for k, v in pce_stds_b[i].items():
+ if k not in std_all:
+ std_all[k] = [None] * len(pce_stds_b)
+ std_all[k][i] = v
+
+ # Back transformation if PCA is selected.
+ pce_means, pce_stds = {}, {}
+ for output in outputs:
+ pce_means[output] = np.mean(mean_all[output], axis=0)
+ pce_stds[output] = np.mean(std_all[output], axis=0)
+
+ return pce_means, pce_stds
diff --git a/src/bayesvalidrox/surrogate_models/surrogate_models.py b/src/bayesvalidrox/surrogate_models/surrogate_models.py
index ca902f26bef0c45e8befb72ff67313ef09a77603..a13a96cc53da289b08f05683234c7d7203b6ed8e 100644
--- a/src/bayesvalidrox/surrogate_models/surrogate_models.py
+++ b/src/bayesvalidrox/surrogate_models/surrogate_models.py
@@ -151,6 +151,7 @@ class MetaModel():
if not hasattr(self, 'CollocationPoints'):
raise AttributeError('Please provide samples to the metamodel before building it.')
+ self.CollocationPoints = np.array(self.CollocationPoints)
# Transform input samples
# TODO: this is probably not yet correct! Make 'method' variable
@@ -220,6 +221,10 @@ class MetaModel():
None.
"""
+# print(X)
+# print(X.shape)
+# print(y)
+# print(y['Z'].shape)
X = np.array(X)
for key in y.keys():
y_val = np.array(y[key])
@@ -301,6 +306,7 @@ class MetaModel():
if fast_bootstrap and b_i == 0:
n_comp_dict[key] = n_comp
else:
+ #print(b_indices)
target = Output[b_indices]
# Parallel fit regression
@@ -1288,7 +1294,7 @@ class MetaModel():
return mean_pred, std_pred
# -------------------------------------------------------------------------
- def create_model_error(self, X, y, Model, name='Calib'):
+ def create_model_error(self, X, y, MeasuredData, name='Calib'):
"""
Fits a GPE-based model error.
@@ -1315,7 +1321,7 @@ class MetaModel():
# Read data
# TODO: do this call outside the metamodel
- MeasuredData = Model.read_observation(case=name)
+ #MeasuredData = Model.read_observation(case=name)
# Fitting GPR based bias model
for out in outputNames:
diff --git a/tests/test_engine.py b/tests/itest_Engine.py
similarity index 100%
rename from tests/test_engine.py
rename to tests/itest_Engine.py
diff --git a/tests/test_MetaModel.py b/tests/itest_MetaModel.py
similarity index 100%
rename from tests/test_MetaModel.py
rename to tests/itest_MetaModel.py
diff --git a/tests/test_BayesInference.py b/tests/test_BayesInference.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fbb79195799724a8ae820a2c777c99c0b86d9b0
--- /dev/null
+++ b/tests/test_BayesInference.py
@@ -0,0 +1,422 @@
+# -*- coding: utf-8 -*-
+"""
+Test the BayesInference class for bayesvalidrox
+
+Tests are available for the following functions
+ _logpdf - x
+ _kernel_rbf - x
+class BayesInference:
+ create_inference
+ perform_bootstrap
+ _perturb_data - x
+ _eval_model Need working model to test this
+ normpdf
+ _corr_factor_BME_old - removed
+ _corr_factor_BME - x
+ _rejection_sampling - x
+ _posterior_predictive
+ _plot_max_a_posteriori
+ _plot_post_predictive
+"""
+import sys
+sys.path.append("src/")
+sys.path.append("../src/")
+import pytest
+import numpy as np
+import pandas as pd
+
+from bayesvalidrox.surrogate_models.inputs import Input
+from bayesvalidrox.surrogate_models.exp_designs import ExpDesigns
+from bayesvalidrox.surrogate_models.surrogate_models import MetaModel
+from bayesvalidrox.pylink.pylink import PyLinkForwardModel as PL
+from bayesvalidrox.surrogate_models.engine import Engine
+from bayesvalidrox.bayes_inference.discrepancy import Discrepancy
+from bayesvalidrox.bayes_inference.mcmc import MCMC
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+from bayesvalidrox.bayes_inference.bayes_inference import _logpdf, _kernel_rbf
+
+#%% Test _logpdf
+
+def test_logpdf() -> None:
+ """
+ Calculate loglikelihood
+
+ """
+ _logpdf([0],[0],[1])
+
+#%% Test _kernel_rbf
+
+def test_kernel_rbf() -> None:
+ """
+ Create RBF kernel
+ """
+ X = [[0,0],[1,1.5]]
+ pars = [1,0.5,1]
+ _kernel_rbf(X, pars)
+
+def test_kernel_rbf_lesspar() -> None:
+ """
+ Create RBF kernel with too few parameters
+ """
+ X = [[0,0],[1,1.5]]
+ pars = [1,2]
+ with pytest.raises(AttributeError) as excinfo:
+ _kernel_rbf(X, pars)
+ assert str(excinfo.value) == 'Provide 3 parameters for the RBF kernel!'
+
+#%% Test MCMC init
+
+def test_BayesInference() -> None:
+ """
+ Construct a BayesInference object
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ engine = Engine(mm, mod, expdes)
+ BayesInference(engine)
+
+#%% Test create_inference
+# TODO: disabled this test!
+def itest_create_inference() -> None:
+ """
+ Run inference
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+
+ expdes = ExpDesigns(inp)
+ expdes.n_init_samples = 2
+ expdes.n_max_samples = 4
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':[[0.4],[0.5],[0.45]]}
+ expdes.x_values = np.array([0]) # Error in plots if this is not available
+
+ mm = MetaModel(inp)
+ mm.fit(expdes.X, expdes.Y)
+ expdes.generate_ED(expdes.n_init_samples, transform=True, max_pce_deg=np.max(mm.pce_deg))
+
+ mod = PL()
+ mod.observations = {'Z':np.array([0.45])}
+ mod.observations = {'Z':np.array([0.45]), 'x_values':np.array([0])} # Error if x_values not given
+ mod.Output.names = ['Z']
+
+ engine = Engine(mm, mod, expdes)
+
+ sigma2Dict = {'Z':np.array([0.05])}
+ sigma2Dict = pd.DataFrame(sigma2Dict, columns = ['Z'])
+ obsData = pd.DataFrame(mod.observations, columns=mod.Output.names)
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = (obsData*0.15)**2
+
+ bi = BayesInference(engine)
+ bi.Discrepancy = DiscrepancyOpts # Error if this not class 'DiscrepancyOpts' or dict(?)
+ bi.bootstrap = True # Error if this and bayes_loocv and just_analysis are all False?
+ bi.plot_post_pred = False # Remaining issue in the violinplot
+ bi.create_inference()
+ # Remaining issue in the violinplot in plot_post_predictive
+
+
+#%% Test rejection_sampling
+def test_rejection_sampling_nologlik() -> None:
+ """
+ Perform rejection sampling without given log likelihood
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ engine = Engine(mm, mod, expdes)
+ bi = BayesInference(engine)
+ bi.prior_samples = expdes.generate_samples(100, 'random')
+ with pytest.raises(AttributeError) as excinfo:
+ bi._rejection_sampling()
+ assert str(excinfo.value) == 'No log-likelihoods available!'
+
+def test_rejection_sampling_noprior() -> None:
+ """
+ Perform rejection sampling without prior samples
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ engine = Engine(mm, mod, expdes)
+ bi = BayesInference(engine)
+ with pytest.raises(AttributeError) as excinfo:
+ bi._rejection_sampling()
+ assert str(excinfo.value) == 'No prior samples available!'
+
+def test_rejection_sampling() -> None:
+ """
+ Perform rejection sampling
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ engine = Engine(mm, mod, expdes)
+ bi = BayesInference(engine)
+ bi.prior_samples = expdes.generate_samples(100, 'random')
+ bi.log_likes = np.swapaxes(np.atleast_2d(np.log(np.random.random(100)*3)),0,1)
+ bi._rejection_sampling()
+
+
+#%% Test _perturb_data
+
+def test_perturb_data() -> None:
+ """
+ Perturb data
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ engine = Engine(mm, mod, expdes)
+
+ bi = BayesInference(engine)
+ data = pd.DataFrame()
+ data['Z'] = [0.45]
+ bi._perturb_data(data, ['Z'])
+
+
+def test_perturb_data_loocv() -> None:
+ """
+ Perturb data with bayes_loocv
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ mod = PL()
+ mm = MetaModel(inp)
+ expdes = ExpDesigns(inp)
+ engine = Engine(mm, mod, expdes)
+
+ bi = BayesInference(engine)
+ data = pd.DataFrame()
+ data['Z'] = [0.45]
+ bi.bayes_loocv = True
+ bi._perturb_data(data, ['Z'])
+
+#%% Test _eval_model
+
+def test_eval_model() -> None:
+ """
+ Run model with descriptive key
+ """
+ # TODO: need functioning example model to test this
+ None
+
+#%% Test corr_factor_BME
+
+def test_corr_factor_BME() -> None:
+ """
+ Calculate correction factor
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':[[0.4],[0.5],[0.45]]}
+
+ mm = MetaModel(inp)
+ mm.fit(expdes.X, expdes.Y)
+ mod = PL()
+ engine = Engine(mm, mod, expdes)
+
+ obs_data = {'Z':np.array([0.45])}
+ total_sigma2s = {'Z':np.array([0.15])}
+ logBME = [0,0,0]
+
+ bi = BayesInference(engine)
+ bi.selected_indices = {'Z':0}
+ bi. _corr_factor_BME(obs_data, total_sigma2s, logBME)
+
+def test_corr_factor_BME_selectedindices() -> None:
+ """
+ Calculate correction factor
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':[[0.4],[0.5],[0.45]]}
+
+ mm = MetaModel(inp)
+ mm.fit(expdes.X, expdes.Y)
+ mod = PL()
+ engine = Engine(mm, mod, expdes)
+
+ obs_data = {'Z':np.array([0.45])}
+ total_sigma2s = {'Z':np.array([0.15])}
+ logBME = [0,0,0]
+
+ bi = BayesInference(engine)
+ bi.selected_indices = {'Z':0}
+ bi. _corr_factor_BME(obs_data, total_sigma2s, logBME)
+
+
+#%% Test normpdf
+
+def test_normpdf_nosigmas() -> None:
+ """
+ Run normpdf without any additional sigmas
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':np.array([[0.4],[0.5],[0.45]])}
+
+ mm = MetaModel(inp)
+ mod = PL()
+ mod.Output.names = ['Z']
+ engine = Engine(mm, mod, expdes)
+
+ obs_data = {'Z':np.array([0.45])}
+ total_sigma2s = {'Z':np.array([0.15])}
+
+ bi = BayesInference(engine)
+ bi.normpdf(expdes.Y, obs_data, total_sigma2s, sigma2=None, std=None)
+
+def test_normpdf_sigma2() -> None:
+ """
+ Run normpdf with sigma2
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':np.array([[0.4],[0.5],[0.45]])}
+
+ mm = MetaModel(inp)
+ mod = PL()
+ mod.Output.names = ['Z']
+ engine = Engine(mm, mod, expdes)
+
+ obs_data = {'Z':np.array([0.45])}
+ total_sigma2s = {'Z':np.array([0.15])}
+ sigma2 = [[0]]
+
+ bi = BayesInference(engine)
+ bi.normpdf(expdes.Y, obs_data, total_sigma2s, sigma2=sigma2, std=None)
+
+def test_normpdf_allsigmas() -> None:
+ """
+ Run normpdf with all additional sigmas
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ expdes = ExpDesigns(inp)
+ expdes.init_param_space(max_deg=1)
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':np.array([[0.4],[0.5],[0.45]])}
+
+ mm = MetaModel(inp)
+ mod = PL()
+ mod.Output.names = ['Z']
+ engine = Engine(mm, mod, expdes)
+
+ obs_data = {'Z':np.array([0.45])}
+ total_sigma2s = {'Z':np.array([0.15])}
+ sigma2 = [[0]]
+
+ bi = BayesInference(engine)
+ bi.normpdf(expdes.Y, obs_data, total_sigma2s, sigma2=sigma2, std=total_sigma2s)
+
+
+#%% Test perform_bootstrap
+
+def test_perform_bootstrap() -> None:
+ """
+ Do bootstrap
+ """
+
+ opt_sigma = 'B'
+ total_sigma2s = {'Z':np.array([0.15])}
+ bi.perform_bootstrap(opt_sigma, total_sigma2s)
+
+if __name__ == '__main__':
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+
+ expdes = ExpDesigns(inp)
+ expdes.n_init_samples = 2
+ expdes.n_max_samples = 4
+ expdes.X = np.array([[0],[1],[0.5]])
+ expdes.Y = {'Z':[[0.4],[0.5],[0.45]]}
+ expdes.x_values = np.array([0]) # Error in plots if this is not available
+
+ mm = MetaModel(inp)
+ mm.n_params = 1
+ mm.fit(expdes.X, expdes.Y)
+ expdes.generate_ED(expdes.n_init_samples, transform=True, max_pce_deg=np.max(mm.pce_deg))
+
+ mod = PL()
+ mod.observations = {'Z':np.array([0.45])}
+ mod.observations = {'Z':np.array([0.45]), 'x_values':np.array([0])} # Error if x_values not given
+ mod.Output.names = ['Z']
+
+ engine = Engine(mm, mod, expdes)
+
+ sigma2Dict = {'Z':np.array([0.05])}
+ sigma2Dict = pd.DataFrame(sigma2Dict, columns = ['Z'])
+ obsData = pd.DataFrame(mod.observations, columns=mod.Output.names)
+ DiscrepancyOpts = Discrepancy('')
+ DiscrepancyOpts.type = 'Gaussian'
+ DiscrepancyOpts.parameters = (obsData*0.15)**2
+
+ bi = BayesInference(engine)
+ bi.Discrepancy = DiscrepancyOpts # Error if this not class 'DiscrepancyOpts' or dict(?)
+ bi.bootstrap = True # Error if this and bayes_loocv and just_analysis are all False?
+ bi.plot_post_pred = False # Remaining issue in the violinplot
+ #bi.error_model = True
+ #bi.bayes_loocv = True
+ bi.create_inference()
+ stop
+ opt_sigma = 'B'
+ total_sigma2s = {'Z':np.array([0.15])}
+ data = pd.DataFrame()
+ data['Z'] = [0.45]
+ data['x_values'] = [0.3]
+ bi.measured_data = data
+ bi.perform_bootstrap(opt_sigma, total_sigma2s)
+
\ No newline at end of file
diff --git a/tests/test_BayesModelComparison.py b/tests/test_BayesModelComparison.py
new file mode 100644
index 0000000000000000000000000000000000000000..91f328ec7ae39cf7cded6b228edc1442053a2dfb
--- /dev/null
+++ b/tests/test_BayesModelComparison.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+"""
+Test the BayesModelComparison class in bayesvalidrox.
+Tests are available for the following functions
+Class BayesModelComparison:
+ create_model_comparison
+ compare_models
+ generate_dataset
+ __perturb_data
+ cal_model_weight
+ plot_just_analysis
+ plot_model_weights
+ plot_bayes_factor
+
+"""
+import sys
+sys.path.append("src/")
+import pytest
+import numpy as np
+
+from bayesvalidrox.bayes_inference.bayes_model_comparison import BayesModelComparison
+#from bayesvalidrox.surrogate_models.input_space import InputSpace
+
+def test_BMC() -> None:
+ """
+ Build BMC without inputs
+ """
+ BayesModelComparison()
\ No newline at end of file
diff --git a/tests/test_Discrepancy.py b/tests/test_Discrepancy.py
index c46e0a13751756e0583f3176489e7215da77f4ba..7fb948d905031e7d7e6235857c27792fb29ece57 100644
--- a/tests/test_Discrepancy.py
+++ b/tests/test_Discrepancy.py
@@ -36,22 +36,8 @@ def test_get_sample() -> None:
"""
Get discrepancy sample
"""
- inp = Input()
- inp.add_marginals()
- inp.Marginals[0].dist_type = 'normal'
- inp.Marginals[0].parameters = [0,1]
- disc = Discrepancy(InputDisc = inp)
+ disc = Discrepancy()
with pytest.raises(AttributeError) as excinfo:
disc.get_sample(2)
assert str(excinfo.value) == 'Cannot create new samples, please provide input distributions'
-
-
-
-
-if __name__ == '__main__':
- inp = Input()
- inp.add_marginals()
- inp.Marginals[0].dist_type = 'normal'
- inp.Marginals[0].parameters = [0,1]
- disc = Discrepancy(InputDisc = inp)
- disc.get_sample(2)
\ No newline at end of file
+
\ No newline at end of file
diff --git a/tests/test_ExpDesign.py b/tests/test_ExpDesign.py
index 42f87663c2d843c4fa3a23e047270673501dbd4c..8b8618e5447870bea2afbd5c9148fa87ddab7b52 100644
--- a/tests/test_ExpDesign.py
+++ b/tests/test_ExpDesign.py
@@ -131,6 +131,47 @@ def test_random_sampler() -> None:
exp = ExpDesigns(inp)
exp.random_sampler(4)
+def test_random_sampler_largedatanoJDist() -> None:
+ """
+ Sample randomly, init_param_space implicitly, more samples wanted than given, no JDist available
+ """
+ x = np.random.uniform(0,1,1000)
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].input_data = x
+ exp = ExpDesigns(inp)
+ with pytest.raises(AttributeError) as excinfo:
+ exp.random_sampler(100000)
+ assert str(excinfo.value) == 'Sampling cannot proceed, build ExpDesign with max_deg != 0 to create JDist!'
+
+def test_random_sampler_largedataJDist0() -> None:
+ """
+ Sample randomly, init_param_space implicitly, more samples wanted than given,
+ JDist available, priors given via samples
+ """
+ x = np.random.uniform(0,1,1000)
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].input_data = x
+ exp = ExpDesigns(inp)
+ exp.init_param_space(max_deg = 1)
+ exp.random_sampler(100000)
+
+def test_random_sampler_largedataJDist1() -> None:
+ """
+ Sample randomly, init_param_space implicitly, more samples wanted than given,
+ JDist available, prior distributions given
+ """
+ inp = Input()
+ inp.add_marginals()
+ inp.Marginals[0].dist_type = 'normal'
+ inp.Marginals[0].parameters = [0,1]
+ exp = ExpDesigns(inp)
+ exp.init_param_space(max_deg = 1)
+ exp.random_sampler(100000)
+
+
+
def test_random_sampler_rawdata() -> None:
"""
Sample randomly, init_param_space implicitly, has 2d raw data
diff --git a/tests/test_InputSpace.py b/tests/test_InputSpace.py
index 1b5a28fa3eb4b1ad11c8a666a9e98e2b0dbaa8b9..719336e88e285b3c4bbc61cc1704c0d0c1fa64bf 100644
--- a/tests/test_InputSpace.py
+++ b/tests/test_InputSpace.py
@@ -11,6 +11,7 @@ Class InputSpace:
"""
import sys
sys.path.append("src/")
+sys.path.append("../src/")
import pytest
import numpy as np
@@ -577,4 +578,3 @@ def test_transform_gammaparam() -> None:
exp = InputSpace(inp)
exp.init_param_space(max_deg=2)
exp.transform(y, params = [1,1])
-
\ No newline at end of file
diff --git a/tests/test_MCMC.py b/tests/test_MCMC.py
new file mode 100644
index 0000000000000000000000000000000000000000..50f6d69b67449ef0647e777f876dd10210566d8f
--- /dev/null
+++ b/tests/test_MCMC.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""
+Test the MCM class of bayesvalidrox
+Tests are available for the following functions
+Class MCMC:
+ run_sampler
+ log_prior
+ log_likelihood
+ log_posterior
+ eval_model
+ train_error_model
+ gelmain_rubin
+ marginal_llk_emcee
+ _iterative_scheme
+ _my_ESS
+ _check_ranges
+"""
+import sys
+sys.path.append("src/")
+sys.path.append("../src/")
+import pytest
+import numpy as np
+
+from bayesvalidrox.bayes_inference.mcmc import MCMC
+from bayesvalidrox.bayes_inference.bayes_inference import BayesInference
+
+#%% Test MCMC init
+
+def test_MCMC() -> None:
+ """
+ Construct an MCMC object
+ """
+ MCMC('')
+
+def test_MCMC() -> None:
+ """
+ Construct an MCMC object
+ """
+ MCMC('')
+
+
+
+
+