{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Loading BOA from JSON and Plotting Results\n", "\n", "This notebook demonstrates how to:\n", "\n", "Loading a Scheduler from JSON from a previous run (If you want to see the Experiment that this is from, see :ref:`Running BOA Optimization Directly in Python`). We will look at the output and plot some exploratory data analysis." ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "tags": [] }, "outputs": [ { "data": { "text/html": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 11-30 12:47:43] ax.utils.notebook.plotting: Injecting Plotly library into cell. Do not overwrite or delete cell.\n" ] }, { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import pathlib\n", "import os\n", "\n", "import numpy as np\n", "from ax.utils.notebook.plotting import init_notebook_plotting\n", "from ax.plot.trace import optimization_trace_single_method_plotly\n", "from ax.service.utils.report_utils import get_standard_plots, exp_to_df\n", "import boa\n", "from botorch.test_functions.synthetic import Cosine8\n", "\n", "init_notebook_plotting()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Loading the Scheduler" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "# Filepath to the scheduler.json\n", "scheduler_fp = pathlib.Path(os.path.abspath(\"\")) / \"boa_runs/scheduler.json\"" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "jupyter": { "outputs_hidden": false } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 11-30 12:47:44] Scheduler: Experiment(boa_runs) already has 100 trials associated with it. Total trials setting for this scheduler is 100, so no more trials would be run in this scheduler if `Scheduler.run_all_trials` is called (but you can still use `Scheduler.run_n_trials` to run a fixed number of trials).\n" ] } ], "source": [ "scheduler = boa.scheduler_from_json_file(scheduler_fp)\n", "experiment = scheduler.experiment" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get the Best Trial and Output All Trials" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "jupyter": { "outputs_hidden": false } }, "outputs": [ { "data": { "text/plain": [ "99" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "best_trial, best_params, obj = scheduler.get_best_trial()\n", "best_trial" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "jupyter": { "outputs_hidden": false } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
Cosine8trial_indexarm_namex0x1x2x3x4x5x6x7trial_statusgeneration_method
02.16712400_00.7745440.4688050.0680630.3623160.8795790.2736480.8329610.104289COMPLETEDSobol
11.33269211_00.4909800.1225340.2860720.1355330.4072120.5164110.5529750.209887COMPLETEDSobol
22.23977922_00.3705150.4378680.6132050.6304940.1351830.4723510.5525780.686980COMPLETEDSobol
33.41718433_00.1368360.6952780.9362320.8445620.0991370.9714490.4679570.413035COMPLETEDSobol
44.00973944_00.7470550.2032360.9846400.2578930.5422770.7426400.6820410.910413COMPLETEDSobol
..........................................
953.1579639595_00.4281700.8751730.9481100.4396050.6634190.4385040.6039670.475487COMPLETEDGPEI
961.8032109696_00.5161410.6350450.3597970.2229310.5879800.1717130.4711700.425680COMPLETEDGPEI
972.4267939797_00.3857000.4148250.1686690.7689330.4463030.6648910.7974530.790683COMPLETEDGPEI
983.7708189898_00.5686600.7639750.8165660.9320600.5710920.5173280.8336060.391031COMPLETEDGPEI
993.3414489999_00.7792550.2451770.9755800.8670210.3154990.4686690.4967130.597464COMPLETEDGPEI
\n", "

100 rows × 13 columns

\n", "
" ], "text/plain": [ " Cosine8 trial_index arm_name x0 x1 x2 x3 \\\n", "0 2.167124 0 0_0 0.774544 0.468805 0.068063 0.362316 \n", "1 1.332692 1 1_0 0.490980 0.122534 0.286072 0.135533 \n", "2 2.239779 2 2_0 0.370515 0.437868 0.613205 0.630494 \n", "3 3.417184 3 3_0 0.136836 0.695278 0.936232 0.844562 \n", "4 4.009739 4 4_0 0.747055 0.203236 0.984640 0.257893 \n", ".. ... ... ... ... ... ... ... \n", "95 3.157963 95 95_0 0.428170 0.875173 0.948110 0.439605 \n", "96 1.803210 96 96_0 0.516141 0.635045 0.359797 0.222931 \n", "97 2.426793 97 97_0 0.385700 0.414825 0.168669 0.768933 \n", "98 3.770818 98 98_0 0.568660 0.763975 0.816566 0.932060 \n", "99 3.341448 99 99_0 0.779255 0.245177 0.975580 0.867021 \n", "\n", " x4 x5 x6 x7 trial_status generation_method \n", "0 0.879579 0.273648 0.832961 0.104289 COMPLETED Sobol \n", "1 0.407212 0.516411 0.552975 0.209887 COMPLETED Sobol \n", "2 0.135183 0.472351 0.552578 0.686980 COMPLETED Sobol \n", "3 0.099137 0.971449 0.467957 0.413035 COMPLETED Sobol \n", "4 0.542277 0.742640 0.682041 0.910413 COMPLETED Sobol \n", ".. ... ... ... ... ... ... \n", "95 0.663419 0.438504 0.603967 0.475487 COMPLETED GPEI \n", "96 0.587980 0.171713 0.471170 0.425680 COMPLETED GPEI \n", "97 0.446303 0.664891 0.797453 0.790683 COMPLETED GPEI \n", "98 0.571092 0.517328 0.833606 0.391031 COMPLETED GPEI \n", "99 0.315499 0.468669 0.496713 0.597464 COMPLETED GPEI \n", "\n", "[100 rows x 13 columns]" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "exp_to_df(experiment)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## EDA Plots" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "jupyter": { "outputs_hidden": false } }, "outputs": [], "source": [ "plots = get_standard_plots(experiment, scheduler.generation_strategy.model)\n", "for plot in plots:\n", " plot.show()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "optimum = -Cosine8().optimal_value\n", "model_transitions = scheduler.generation_strategy.model_transitions\n", "data = experiment.fetch_data()\n", "metric_name = \"Cosine8\"\n", "\n", "optimization_trace_single_method_plotly(\n", " y=np.array([data.df[data.df[\"metric_name\"] == metric_name][\"mean\"]]),\n", " title=r\"$\\text{ Cosine Mixture: } f(x) = -0.1 \\sum_{i=1}^n (cos(5 \\pi x_i) - x_i^2) : x \\in [-1, 1]^8$\",\n", " ylabel=\"Cosine Mixture\",\n", " model_transitions=model_transitions,\n", " # Try and use the metric's lower_is_better property, but fall back on\n", " # objective's minimize property if relevent\n", " optimum=optimum,\n", " optimization_direction=(\n", " (\"minimize\" if experiment.metrics[metric_name].lower_is_better is True else \"maximize\")\n", " if experiment.metrics[metric_name].lower_is_better is not None\n", " else (\"minimize\" if optimization_config.objective.minimize else \"maximize\")\n", " ),\n", " plot_trial_points=True,\n", ")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 4 }