datagenerator.Parameters

   1from collections import defaultdict
   2import datetime
   3import json
   4import multiprocessing as mp
   5import os
   6import pathlib
   7import glob
   8import sqlite3
   9from subprocess import CalledProcessError
  10import numpy as np
  11import tables
  12import subprocess
  13
  14dir_name = os.path.dirname(__file__)
  15CONFIG_PATH = os.path.abspath(os.path.join(dir_name, "../config/config_ht.json"))
  16
  17
  18class _Borg:
  19    # Any objects using Borg base class will have same shared_state values (Alex Martelli's Borg)
  20    _shared_state = {}
  21
  22    def __init__(self):
  23        self.__dict__ = self._shared_state
  24
  25
  26class Parameters(_Borg):
  27    """
  28    Parameter object storing all model parameters.
  29
  30    Attributes
  31    ----------
  32    model_dir_name : str
  33        This is the name that the directory will be given, by default `seismic`
  34    parameter_file : str
  35        User parameters are read from the 'user_config' json file, and
  36        additional model parameters are set.
  37    test_mode : int
  38        If test_mode is set using an integer, the size of the model will
  39        be reduced e.g 100 makes a 100x100.It reduces the ammount of 
  40        time that the program takes to generate data usueful when testing
  41        a model.
  42
  43        **Warning: If you put too small a number, the model may fail due to 
  44        not enough space to place faults etc...**
  45
  46        Value should ideally be >= 50
  47    runid : str
  48        The string runid will be added to the final model directory.
  49    rpm_scaling_factors : dict
  50        These are user-defined parameter. You can use the defaults
  51        provided, but results might be unrealistic. 
  52        These might need to be tuned to get reaslistic synthetic
  53        data.
  54    sqldict : dict
  55        This is a dictionary structure that stores all the parameters
  56        of the model. This dictionary eventually gets written to a
  57        sqlite DB file.
  58
  59    Methods
  60    -------
  61    setup_model(rpm_factors=None) -> None:
  62        Method to set up all the necesary parameters to start a new model.
  63    make_directories() -> None:
  64        Method that generates all necessary directory structures on disk
  65    write_key_file():
  66        Method to generate a key file that describes coordinate systems, 
  67        track, bin, digi (inline, xlines, )
  68    write_to_logfile(msg, mainkey=None, subkey=None, val=""):
  69        Method that writes to the logfile
  70    """
  71
  72    def __init__(self, user_config: str = CONFIG_PATH, test_mode=None, runid=None):
  73        """
  74        Initialize the Parameters object.
  75
  76        Parameters
  77        ----------
  78        user_config : `str`, optional
  79            This is the path on disk that points to a `.json` file
  80            that contains the configurations for each run, by default CONFIG_PATH
  81        test_mode : `int`, optional
  82            The parameter that sets the running mode, by default 0
  83        runid : `str`, optional
  84            This is the runid of the run, this comes in handy when you have many runs
  85            with various permutations of parameters, by default None
  86        """
  87        # reset the parameter dict in case we are building models within a loop, and the shared_state dict is not empty
  88        self._shared_state = {}
  89        super().__init__()
  90        self.model_dir_name: str = "seismic"
  91        self.parameter_file = user_config
  92        self.test_mode = test_mode
  93        self.runid = runid
  94        self.rpm_scaling_factors = dict()
  95        self.sqldict = defaultdict(dict)
  96
  97    def __repr__(self):
  98        """
  99        Representation method
 100
 101        Parameters
 102        ----------
 103        self : `Parameters`
 104            The instance of the Parameters object
 105        """
 106        # Make nice repr instead of a print method
 107        items = ("\t{} = {}".format(k, v) for k, v in self.__dict__.items())
 108        return "{}:\n{}".format(self.__class__.__name__, "\n".join(sorted(items)))
 109
 110    def __getitem__(self, key: str):
 111        """__getitem__
 112
 113        Enable retrieval of values as though the class instance is a dict
 114
 115        Parameters
 116        ----------
 117        key : str
 118            The key desired to be accessed
 119
 120        Returns
 121        -------
 122        any
 123            Value of the key
 124        """
 125        return self._shared_state[key]
 126
 127    def setup_model(self, rpm_factors=None) -> None:
 128        """
 129        Setup Model
 130        -----------
 131        Sets up the creation of essential parameters and directories
 132
 133        Parameters
 134        ----------
 135        rpm_factors : `dict`, optional
 136            The rock physics model factors for generating the synthetic cube.
 137            By default the rpm factors come from a default in the main.py file
 138
 139        Returns
 140        -------
 141        None
 142        """
 143        # Set model parameters
 144        self._set_model_parameters(self.model_dir_name)
 145        self.make_directories()
 146        self.write_key_file()
 147        self._setup_rpm_scaling_factors(rpm_factors)
 148
 149        # Write model parameters to logfile
 150        self._write_initial_model_parameters_to_logfile()
 151
 152    def make_directories(self) -> None:
 153        """
 154        Make directories.
 155        -----------------
 156
 157        Creates the necessary directories to run the model.
 158
 159        This function creates the directories on disk
 160        necessary for the model to run.
 161
 162        Parameters
 163        ----------
 164        self : `Parameters`
 165
 166        Returns
 167        -------
 168        None
 169        """
 170        print(f"\nModel folder: {self.work_subfolder}")
 171        self.sqldict["model_id"] = pathlib.Path(self.work_subfolder).name
 172        for folder in [self.project_folder, self.work_subfolder, self.temp_folder]:
 173            try:
 174                os.stat(folder)
 175            except OSError:
 176                print(f"Creating directory: {folder}")
 177                # Try making folders (can fail if multiple models are being built simultaneously in a new dir)
 178                try:
 179                    os.mkdir(folder)
 180                except OSError:
 181                    pass
 182        try:
 183            os.system(f"chmod -R 777 {self.work_subfolder}")
 184        except OSError:
 185            print(f"Could not chmod {self.work_subfolder}. Continuing...")
 186            pass
 187
 188    def write_key_file(self) -> None:
 189        """
 190        Write key file
 191        --------------
 192
 193        Writes a file that ocntains important parameters about the cube.
 194
 195        Method that writes important parameters about the synthetic cube
 196        such as coordinate transforms and sizes.
 197
 198        Parameters
 199        ----------
 200        None
 201
 202        Returns
 203        ----------
 204        None
 205        """
 206        # Set plausible key file values
 207        geom_expand = dict()
 208        geom_expand["3D_NAME"] = "synthetic data for training"
 209        geom_expand["COORD_METHOD"] = 1
 210        geom_expand["DATA_TYPE"] = "3D"
 211        geom_expand["DELTA_BIN_NUM"] = 1
 212        geom_expand["DELTA_TRACK_NUM"] = 1
 213        geom_expand["DIGITIZATION"] = 4
 214        geom_expand["EPSG_CRS"] = 32066
 215        geom_expand["FIRST_BIN"] = 1000
 216        geom_expand["FIRST_TRACK"] = 2000
 217        geom_expand["FORMAT"] = 1
 218        geom_expand["N_BIN"] = self.cube_shape[1]
 219        geom_expand["N_SAMP"] = self.cube_shape[2]
 220        geom_expand["N_TRACK"] = self.cube_shape[0]
 221        geom_expand["PROJECTION"] = 316
 222        geom_expand["REAL_DELTA_X"] = 100.0
 223        geom_expand["REAL_DELTA_Y"] = 100.0
 224        geom_expand["REAL_GEO_X"] = 1250000.0
 225        geom_expand["REAL_GEO_Y"] = 10500000.0
 226        geom_expand["SKEW_ANGLE"] = 0.0
 227        geom_expand["SUBPOINT_CODE"] = "TTTBBB"
 228        geom_expand["TIME_OR_DEPTH"] = "TIME"
 229        geom_expand["TRACK_DIR"] = "H"
 230        geom_expand["XFORM_TO_WGS84"] = 1241
 231        geom_expand["ZERO_TIME"] = 0
 232
 233        # Write the keyfile
 234        outputkey = os.path.join(
 235            self.work_subfolder, "seismicCube_" + self.date_stamp + ".key"
 236        )
 237        with open(outputkey, "w") as key:
 238            key.write(
 239                "{}MESSAGE_FILE\n".format(20 * " ")
 240            )  # spaces are important here.. Require 20 of them
 241            key.write("3D_NAME C %s\n" % geom_expand["3D_NAME"])
 242            key.write("COORD_METHOD I %d\n" % int(geom_expand["COORD_METHOD"]))
 243            key.write("DATA_TYPE C %s\n" % geom_expand["DATA_TYPE"])
 244            key.write("DELTA_BIN_NUM I %d\n" % int(geom_expand["DELTA_BIN_NUM"]))
 245            key.write("DELTA_TRACK_NUM I %d\n" % int(geom_expand["DELTA_TRACK_NUM"]))
 246            key.write("DIGITIZATION I %d\n" % int(geom_expand["DIGITIZATION"]))
 247            key.write("EPSG_CRS I %d\n" % int(geom_expand["EPSG_CRS"]))
 248            key.write("FIRST_BIN I %d\n" % int(geom_expand["FIRST_BIN"]))
 249            key.write("FIRST_TRACK I %d\n" % int(geom_expand["FIRST_TRACK"]))
 250            key.write("FORMAT I %d\n" % int(geom_expand["FORMAT"]))
 251            key.write("N_BIN I %d\n" % int(geom_expand["N_BIN"]))
 252            key.write("N_SAMP I %d\n" % int(geom_expand["N_SAMP"]))
 253            key.write("N_TRACK I %d\n" % int(geom_expand["N_TRACK"]))
 254            key.write("PROJECTION I %d\n" % int(geom_expand["PROJECTION"]))
 255            key.write("REAL_DELTA_X R %f\n" % float(geom_expand["REAL_DELTA_X"]))
 256            key.write("REAL_DELTA_Y R %f\n" % float(geom_expand["REAL_DELTA_Y"]))
 257            key.write("REAL_GEO_X R %f\n" % float(geom_expand["REAL_GEO_X"]))
 258            key.write("REAL_GEO_Y R %f\n" % float(geom_expand["REAL_GEO_Y"]))
 259            key.write("SKEW_ANGLE R %f\n" % float(geom_expand["SKEW_ANGLE"]))
 260            key.write("SUBPOINT_CODE C %s\n" % geom_expand["SUBPOINT_CODE"])
 261            key.write("TIME_OR_DEPTH C %s\n" % geom_expand["TIME_OR_DEPTH"])
 262            key.write("TRACK_DIR C %s\n" % geom_expand["TRACK_DIR"])
 263            key.write("XFORM_TO_WGS84 I %d\n" % int(geom_expand["XFORM_TO_WGS84"]))
 264            key.write("ZERO_TIME I %d\n" % int(geom_expand["ZERO_TIME"]))
 265        print(f"\nKeyfile created at {outputkey}")
 266
 267    def write_to_logfile(self, msg, mainkey=None, subkey=None, val="") -> None:
 268        """
 269        write_to_logfile
 270
 271        Method to write msg to model_parameter file
 272        (includes newline)
 273
 274        Parameters
 275        ----------
 276        msg : `string`
 277        Required string object that will be written tom model parameter file.
 278        mainkey : `string`
 279        String of the key to be written into de sql dictionary.
 280        subkey : `string`
 281        String of the subkey to be written into de sql dictionary.
 282        val : `string`
 283        String of the value that should be written into the sql dictionary.
 284        
 285        Returns
 286        -------
 287        None
 288        """
 289        if msg is not None:
 290            with open(self.logfile, "a") as f:
 291                f.write(f"{msg}\n")
 292        if mainkey is not None:
 293            self.sqldict[mainkey][subkey] = val
 294            # for k, v in self.sqldict.items():
 295            #     print(f"{k}: {v}")
 296
 297    def write_sqldict_to_logfile(self, logfile=None) -> None:
 298        """
 299        write_sqldict_to_logfile
 300
 301        Write the sql dictionary to the logfile
 302
 303        Parameters
 304        ----------
 305        logfile : `string`
 306        The path to the log file. By default None
 307        
 308        Returns
 309        -------
 310        None
 311        """
 312        if logfile is None:
 313            logfile = self.logfile
 314        with open(logfile, "a") as f:
 315            for k, nested in self.sqldict.items():
 316                print(k, file=f)
 317                if k == "model_id":
 318                    print(f"\t{nested}", file=f)
 319                else:
 320                    for subkey, value in nested.items():
 321                        print(f"\t{subkey}: {value}", file=f)
 322                print(file=f)
 323
 324    def write_sqldict_to_db(self) -> None:
 325        """
 326        write_sqldict_to_db
 327
 328        Method to write the sqldict to database sqlite file
 329
 330        Parameters
 331        ----------
 332        None
 333        
 334        Returns
 335        -------
 336        None
 337        """
 338        model_id = pathlib.Path(self.work_subfolder).name
 339        model_parameters = self.sqldict["model_parameters"]
 340        fault_keys = [k for k in self.sqldict.keys() if "fault" in k]
 341        closure_keys = [k for k in self.sqldict.keys() if "closure" in k]
 342
 343        conn = sqlite3.connect(os.path.join(self.work_subfolder, "parameters.db"))
 344        # tables = ["model_parameters", "fault_parameters", "closure_parameters"]
 345        # create tables
 346        sql = f"CREATE TABLE model_parameters (model_id string primary key, {','.join(model_parameters.keys())})"
 347        conn.execute(sql)
 348        # insert model_parameters
 349        columns = "model_id, " + ", ".join(model_parameters.keys())
 350        placeholders = ", ".join("?" * (len(model_parameters) + 1))
 351        sql = f"INSERT INTO model_parameters ({columns}) VALUES ({placeholders})"
 352        values = tuple([model_id] + [str(x) for x in model_parameters.values()])
 353        conn.execute(sql, values)
 354        conn.commit()
 355
 356        # fault parameters
 357        if len(fault_keys) > 0:
 358            f = tuple(self.sqldict[fault_keys[0]].keys())
 359            sql = f"CREATE TABLE fault_parameters ({','.join(f)})"
 360            conn.execute(sql)
 361            columns = ", ".join(self.sqldict[fault_keys[0]].keys())
 362            placeholders = ", ".join("?" * len(self.sqldict[fault_keys[0]].keys()))
 363            for f in fault_keys:
 364                sql = (
 365                    f"INSERT INTO fault_parameters ({columns}) VALUES ({placeholders})"
 366                )
 367                conn.execute(sql, tuple(self.sqldict[f].values()))
 368                conn.commit()
 369
 370        if len(closure_keys) > 0:
 371            c = tuple(self.sqldict[closure_keys[0]].keys())
 372            sql = f"CREATE TABLE closure_parameters ({','.join(c)})"
 373            conn.execute(sql)
 374            columns = ", ".join(self.sqldict[closure_keys[0]].keys())
 375            placeholders = ", ".join("?" * len(self.sqldict[closure_keys[0]].keys()))
 376            for c in closure_keys:
 377                sql = f"INSERT INTO closure_parameters ({columns}) VALUES ({placeholders})"
 378                conn.execute(sql, tuple(self.sqldict[c].values()))
 379                conn.commit()
 380
 381    def _setup_rpm_scaling_factors(self, rpm_factors: dict) -> None:
 382        """
 383        Setup Rock Physics Model scaling factors
 384        ----------------------------------------
 385
 386        Method to initialize all the rock physics model
 387        scaling factors. Method also writes the values to
 388        the model_parameters log file.
 389
 390        Parameters
 391        ----------
 392        TODO remove the default in the main.py or have a single source of truth
 393        rpm_factors : `dict`
 394        Dictionary containing the scaling factors for the RPM.
 395        If no RPM factors are provided, the default values are used.
 396        
 397        Returns
 398        -------
 399        None
 400        """
 401        if rpm_factors and not self.test_mode:
 402            self.rpm_scaling_factors = rpm_factors
 403        else:
 404            # Use defaults for RPM Z-shifts and scaling factors
 405            self.rpm_scaling_factors = dict()
 406            self.rpm_scaling_factors["layershiftsamples"] = int(
 407                np.random.triangular(35, 75, 125)
 408            )
 409            self.rpm_scaling_factors["RPshiftsamples"] = int(
 410                np.random.triangular(5, 11, 20)
 411            )
 412            self.rpm_scaling_factors["shalerho_factor"] = 1.0
 413            self.rpm_scaling_factors["shalevp_factor"] = 1.0
 414            self.rpm_scaling_factors["shalevs_factor"] = 1.0
 415            self.rpm_scaling_factors["sandrho_factor"] = 1.0
 416            self.rpm_scaling_factors["sandvp_factor"] = 1.0
 417            self.rpm_scaling_factors["sandvs_factor"] = 1.0
 418            self.rpm_scaling_factors["nearfactor"] = 1.0
 419            self.rpm_scaling_factors["midfactor"] = 1.0
 420            self.rpm_scaling_factors["farfactor"] = 1.0
 421        # Write factors to logfile
 422        for k, v in self.rpm_scaling_factors.items():
 423            self.write_to_logfile(
 424                msg=f"{k}: {v}", mainkey="model_parameters", subkey=k, val=v
 425            )
 426
 427    def _set_model_parameters(self, dname: str) -> None:
 428        """
 429        Set Model Parameters
 430        ----------------------------------------
 431
 432        Method that sets model parameters from user-provided
 433        config.json file
 434
 435        Parameters
 436        ----------
 437        dname : `str`
 438        Directory name specified in the configuration file,
 439        or the default is used
 440        
 441        Returns
 442        -------
 443        None
 444        """
 445        self.current_dir = os.getcwd()
 446        self.start_time = datetime.datetime.now()
 447        self.date_stamp = self.year_plus_fraction()
 448
 449        # Read from input json
 450        self.parameters_json = self._read_json()
 451        self._read_user_params()
 452
 453        # Directories
 454        model_dir = f"{dname}__{self.date_stamp}"
 455        temp_dir = f"temp_folder__{self.date_stamp}"
 456        self.work_subfolder = os.path.abspath(
 457            os.path.join(self.project_folder, model_dir)
 458        )
 459        self.temp_folder = os.path.abspath(
 460            os.path.join(self.work_folder, f"temp_folder__{self.date_stamp}")
 461        )
 462        if self.runid:
 463            self.work_subfolder = f"{self.work_subfolder}_{self.runid}"
 464            self.temp_folder = f"{self.temp_folder}_{self.runid}"
 465
 466        # Various model parameters, not in config
 467        self.num_lyr_lut = self.cube_shape[2] * 2 * self.infill_factor
 468        # 2500 voxels = 25x25x4m voxels size, 25% porosity and closures > ~40,000 bbl
 469        # Use the minimum voxel count as initial closure size filter
 470        self.closure_min_voxels = min(
 471            self.closure_min_voxels_simple,
 472            self.closure_min_voxels_faulted,
 473            self.closure_min_voxels_onlap,
 474        )
 475        self.order = self.bandwidth_ord
 476
 477        if self.test_mode:
 478            self._set_test_mode(self.test_mode, self.test_mode)
 479
 480        # Random choices are separated into this method
 481        self._randomly_chosen_model_parameters()
 482        # Fault choices
 483        self._fault_settings()
 484
 485        # Logfile
 486        self.logfile = os.path.join(
 487            self.work_subfolder, f"model_parameters_{self.date_stamp}.txt"
 488        )
 489
 490        # HDF file to store various model data
 491        self.hdf_master = os.path.join(
 492            self.work_subfolder, f"seismicCube__{self.date_stamp}.hdf"
 493        )
 494
 495    def _calculate_snr_after_lateral_filter(self, sn_db: float) -> float:
 496        """
 497        Calculate Signal:Noise Ratio after lateral filter
 498        ----------------------------------------
 499
 500        Method that computes the signal to noise ratio after
 501        the lateral filter is applied.
 502
 503        Parameters
 504        ----------
 505        sn_db : `float`
 506            Value of the signal to noise value from the database
 507        
 508        Returns
 509        -------
 510        pre_smear_snr : `float`
 511            Signal to noise ratio after the lateral filter is applied
 512        """
 513        snr_of_lateral_filter = 10 * np.log10(self.lateral_filter_size ** 2)
 514        pre_smear_snr = sn_db - snr_of_lateral_filter
 515        return pre_smear_snr
 516
 517    def _randomly_chosen_model_parameters(self) -> None:
 518        """
 519        Randomly Chosen Model Parameters
 520        ----------------------------------------
 521
 522        Method that sets all randomly chosen model parameters
 523
 524        Parameters
 525        ----------
 526        None
 527        
 528        Returns
 529        -------
 530        None
 531        """
 532        # Initial layer standard deviation
 533        self.initial_layer_stdev = (
 534            np.random.uniform(self.lyr_stdev[0], high=self.lyr_stdev[1])
 535            * self.infill_factor
 536        )
 537
 538        # lateral filter size, either 1x1, 3x3 or 5x5
 539        self.lateral_filter_size = int(np.random.uniform(0, 2) + 0.5) * 2 + 1
 540
 541        # Signal to noise in decibels
 542        sn_db = triangle_distribution_fix(
 543            left=self.snr_db[0], mode=self.snr_db[1], right=self.snr_db[2]
 544        )
 545        # sn_db = np.random.triangular(left=self.snr_db[0], mode=self.snr_db[1], right=self.snr_db[2])
 546        # self.sn_db = self._calculate_snr_after_lateral_filter(sn_db)
 547        self.sn_db = sn_db
 548
 549        # Percentage of layers that are sand
 550        self.sand_layer_pct = np.random.uniform(
 551            low=self.sand_layer_pct_min, high=self.sand_layer_pct_max
 552        )
 553
 554        # Minimum shallowest depth of seabed
 555        if (
 556            len(self.seabed_min_depth) > 1
 557        ):  # if low/high value provided, select a value between these
 558            self.seabed_min_depth = np.random.randint(
 559                low=self.seabed_min_depth[0], high=self.seabed_min_depth[1]
 560            )
 561
 562        # Low/High bandwidth to be used
 563        self.lowfreq = np.random.uniform(self.bandwidth_low[0], self.bandwidth_low[1])
 564        self.highfreq = np.random.uniform(
 565            self.bandwidth_high[0], self.bandwidth_high[1]
 566        )
 567
 568        # Choose whether to add coherent noise
 569        self.add_noise = np.random.choice((0, 1))
 570        if self.add_noise == 1:
 571            self.smiley_or_frowny = np.random.choice((0, 1))
 572            if self.smiley_or_frowny == 1:
 573                self.fnoise = "random_coherent_frowns"
 574                print("Coherent frowns will be inserted")
 575            else:
 576                self.fnoise = "random_coherent_smiles"
 577                print("Coherent smiles will be inserted")
 578        else:
 579            self.fnoise = "random"
 580            print("No coherent noise will be inserted")
 581
 582        # Salt inclusion
 583        # self.include_salt = np.random.choice([True, False], 1, p=[0.5, 0.5])[0]
 584        self.noise_stretch_factor = np.random.uniform(1.15, 1.35)
 585        if self.include_salt:
 586            print(
 587                "Salt will be inserted. noise_stretch_factor = {}".format(
 588                    np.around(self.noise_stretch_factor, 2)
 589                )
 590            )
 591        else:
 592            print("Salt will be NOT be inserted.")
 593
 594    def _read_json(self) -> dict:
 595        # TODO Move this to a separate function in utlis?
 596        """
 597        Read JSON file
 598        ----------------------------------------
 599
 600        Reads a json file on disk and loads it as
 601        dictionary
 602
 603        Parameters
 604        ----------
 605        None
 606        
 607        Returns
 608        -------
 609        config : `dict`
 610            Dictionary with the configuration options
 611        """
 612        with open(self.parameter_file) as f:
 613            config: dict = json.load(f)
 614        return config
 615
 616    def _read_user_params(self) -> None:
 617        """
 618        Read User Params
 619        ----------------------------------------
 620
 621        Takes the read in dictionary of JSON configuration
 622        and reads each parameter and inserts it into the
 623        attributes.
 624
 625        In the end it prints a summary of the parameters
 626        to the console.
 627
 628        Parameters
 629        ----------
 630        None
 631        
 632        Returns
 633        -------
 634        None
 635        """
 636        d = self._read_json()
 637        self.project = d["project"]
 638        self.project_folder = d["project_folder"]
 639        wfolder = d["work_folder"]
 640        if not os.path.exists(wfolder):
 641            wfolder = "/tmp"  # In case work_folder does not exist, use /tmp
 642        self.work_folder = wfolder
 643        # read parameters into Parameter class attributes
 644        self.cube_shape = tuple(d["cube_shape"])
 645        self.incident_angles = tuple(d["incident_angles"])
 646        self.digi = d["digi"]
 647        self.infill_factor = d["infill_factor"]
 648        self.lyr_stdev = d["initial_layer_stdev"]
 649        self.thickness_min = d["thickness_min"]
 650        self.thickness_max = d["thickness_max"]
 651        self.seabed_min_depth = d["seabed_min_depth"]
 652        self.snr_db = d["signal_to_noise_ratio_db"]
 653        # self.random_depth_perturb = d['random_depth_perturb_range']
 654        self.bandwidth_low = d["bandwidth_low"]
 655        self.bandwidth_high = d["bandwidth_high"]
 656        self.bandwidth_ord = d["bandwidth_ord"]
 657        self.dip_factor_max = d["dip_factor_max"]
 658        self.min_number_faults = d["min_number_faults"]
 659        self.max_number_faults = d["max_number_faults"]
 660        self.basin_floor_fans = d["basin_floor_fans"]
 661        self.pad_samples = d["pad_samples"]
 662        self.qc_plots = d["extra_qc_plots"]
 663        self.verbose = d["verbose"]
 664        self.include_channels = d["include_channels"]
 665        self.include_salt = d["include_salt"]
 666        self.max_column_height = d["max_column_height"]
 667        self.closure_types = d["closure_types"]
 668        self.closure_min_voxels_simple = d["min_closure_voxels_simple"]
 669        self.closure_min_voxels_faulted = d["min_closure_voxels_faulted"]
 670        self.closure_min_voxels_onlap = d["min_closure_voxels_onlap"]
 671        self.partial_voxels = d["partial_voxels"]
 672        self.variable_shale_ng = d["variable_shale_ng"]
 673        self.sand_layer_thickness = d["sand_layer_thickness"]
 674        self.sand_layer_pct_min = d["sand_layer_fraction"]["min"]
 675        self.sand_layer_pct_max = d["sand_layer_fraction"]["max"]
 676        self.hdf_store = d["write_to_hdf"]
 677        self.broadband_qc_volume = d["broadband_qc_volume"]
 678        self.model_qc_volumes = d["model_qc_volumes"]
 679        self.multiprocess_bp = d["multiprocess_bp"]
 680
 681        # print em
 682        self.__repr__()
 683
 684    def _set_test_mode(self, size_x: int = 50, size_y: int = 50) -> None:
 685        """
 686        Set test mode
 687        -------------
 688
 689        Sets whether the parameters for testinf mode. If no size integer
 690        is provided is defaults to 50.
 691
 692        This value is a good minimum because it allows for the 3D model
 693        to be able to contain faults and other objects inside.
 694
 695        Parameters
 696        ----------
 697        size_x : `int`
 698        The parameter that sets the size of the model in the x direction
 699        size_y : `int`
 700        The parameter that sets the size of the model in the y direction
 701        
 702        Returns
 703        -------
 704        None
 705        """
 706
 707        # Set output model folder in work_folder location but with same directory name as project_folder
 708        normpath = (
 709            os.path.normpath(self.project_folder) + "_test_mode_"
 710        )  # strip trailing / if added
 711        new_project_folder = os.path.join(self.work_folder, os.path.basename(normpath))
 712        # Put all folders inside project folder for easy deleting
 713        self.work_folder = new_project_folder
 714        self.project_folder = new_project_folder
 715        self.work_subfolder = os.path.join(
 716            new_project_folder, os.path.basename(self.work_subfolder)
 717        )
 718        if self.runid:
 719            # Append runid if provided
 720            self.temp_folder = f"{self.temp_folder}_{self.runid}__{self.date_stamp}"
 721        else:
 722            self.temp_folder = os.path.abspath(
 723                os.path.join(self.work_folder, f"temp_folder__{self.date_stamp}")
 724            )
 725        # Set smaller sized model
 726        self.cube_shape = tuple([size_x, size_y, self.cube_shape[-1]])
 727        # Print message to user
 728        print(
 729            "{0}\nTesting Mode\nOutput Folder: {1}\nCube_Shape: {2}\n{0}".format(
 730                36 * "-", self.project_folder, self.cube_shape
 731            )
 732        )
 733
 734    def _fault_settings(self) -> None:
 735        """
 736        Set Fault Settings
 737        -------------
 738
 739        Sets the parameters that will be used to generate faults throughout
 740        the synthetic model.
 741
 742        Parameters
 743        ----------
 744        None
 745        
 746        Returns
 747        -------
 748        None
 749        """
 750        # Fault parameters
 751        self.low_fault_throw = 5.0 * self.infill_factor
 752        self.high_fault_throw = 35.0 * self.infill_factor
 753
 754        # mode & clustering are randomly chosen
 755        self.mode = np.random.choice([0, 1, 2], 1)[
 756            0
 757        ]
 758        self.clustering = np.random.choice([0, 1, 2], 1)[0]
 759
 760        if self.mode == 0:
 761            # As random as it can be
 762            self.number_faults = np.random.randint(
 763                self.min_number_faults, self.max_number_faults
 764            )
 765            self.fmode = "random"
 766
 767        elif self.mode == 1:
 768            if self.clustering == 0:
 769                self.fmode = "self_branching"
 770                # Self Branching. avoid large fault
 771                self.number_faults = np.random.randint(
 772                    3, 9
 773                )
 774                self.low_fault_throw = 5.0 * self.infill_factor
 775                self.high_fault_throw = 15.0 * self.infill_factor
 776            if self.clustering == 1:
 777                # Stair case
 778                self.fmode = "stair_case"
 779                self.number_faults = np.random.randint(
 780                    5, self.max_number_faults
 781                )
 782            if self.clustering == 2:
 783                # Relay ramps
 784                self.fmode = "relay_ramp"
 785                self.number_faults = np.random.randint(
 786                    3, 9
 787                )
 788                self.low_fault_throw = 5.0 * self.infill_factor
 789                self.high_fault_throw = 15.0 * self.infill_factor
 790        elif self.mode == 2:
 791            # Horst and graben
 792            self.fmode = "horst_and_graben"
 793            self.number_faults = np.random.randint(
 794                3, 7
 795            )
 796
 797        self.fault_param = [
 798            str(self.mode) + str(self.clustering),
 799            self.number_faults,
 800            self.low_fault_throw,
 801            self.high_fault_throw,
 802        ]
 803
 804    def _get_commit_hash(self) -> str:
 805        """
 806        Get Commit Hash
 807        -------------
 808
 809        Gets the commit hash of the current git repository.
 810
 811        #TODO Explain what this is for exactly
 812
 813        Parameters
 814        ----------
 815        None
 816        
 817        Returns
 818        -------
 819        sha : `str`
 820            The commit hash of the current git repository
 821        """
 822
 823        try:
 824            sha = (
 825                subprocess.check_output(["git", "rev-parse", "HEAD"])
 826                .decode("utf-8")
 827                .strip()
 828            )
 829        except CalledProcessError:
 830            sha = "cwd not a git repository"
 831        return sha
 832
 833    def _write_initial_model_parameters_to_logfile(self) -> None:
 834        """
 835        Write Initial Model Parameters to Logfile
 836        ----------------------------------------
 837
 838        Method that writes the initial parameters set for the model
 839        to the logfile.
 840
 841        Parameters
 842        ----------
 843        None
 844        
 845        Returns
 846        -------
 847        None
 848        """
 849        _sha = self._get_commit_hash()
 850        self.write_to_logfile(
 851            f"SHA: {_sha}", mainkey="model_parameters", subkey="sha", val=_sha
 852        )
 853        self.write_to_logfile(
 854            f"modeling start time: {self.start_time}",
 855            mainkey="model_parameters",
 856            subkey="start_time",
 857            val=self.start_time,
 858        )
 859        self.write_to_logfile(
 860            f"project_folder: {self.project_folder}",
 861            mainkey="model_parameters",
 862            subkey="project_folder",
 863            val=self.project_folder,
 864        )
 865        self.write_to_logfile(
 866            f"work_subfolder: {self.work_subfolder}",
 867            mainkey="model_parameters",
 868            subkey="work_subfolder",
 869            val=self.work_subfolder,
 870        )
 871        self.write_to_logfile(
 872            f"cube_shape: {self.cube_shape}",
 873            mainkey="model_parameters",
 874            subkey="cube_shape",
 875            val=self.cube_shape,
 876        )
 877        self.write_to_logfile(
 878            f"incident_angles: {self.incident_angles}",
 879            mainkey="model_parameters",
 880            subkey="incident_angles",
 881            val=self.incident_angles,
 882        )
 883        self.write_to_logfile(
 884            f"number_faults: {self.number_faults}",
 885            mainkey="model_parameters",
 886            subkey="number_faults",
 887            val=self.number_faults,
 888        )
 889        self.write_to_logfile(
 890            f"lateral_filter_size: {self.lateral_filter_size}",
 891            mainkey="model_parameters",
 892            subkey="lateral_filter_size",
 893            val=self.lateral_filter_size,
 894        )
 895        self.write_to_logfile(
 896            f"salt_inserted: {self.include_salt}",
 897            mainkey="model_parameters",
 898            subkey="salt_inserted",
 899            val=self.include_salt,
 900        )
 901        self.write_to_logfile(
 902            f"salt noise_stretch_factor: {self.noise_stretch_factor:.2f}",
 903            mainkey="model_parameters",
 904            subkey="salt_noise_stretch_factor",
 905            val=self.noise_stretch_factor,
 906        )
 907        self.write_to_logfile(
 908            f"bandpass_bandlimits: {self.lowfreq:.2f}, {self.highfreq:.2f}"
 909        )
 910        self.write_to_logfile(
 911            msg=None,
 912            mainkey="model_parameters",
 913            subkey="bandpass_bandlimit_low",
 914            val=self.lowfreq,
 915        )
 916        self.write_to_logfile(
 917            msg=None,
 918            mainkey="model_parameters",
 919            subkey="bandpass_bandlimit_high",
 920            val=self.highfreq,
 921        )
 922        self.write_to_logfile(
 923            f"sn_db: {self.sn_db:.2f}",
 924            mainkey="model_parameters",
 925            subkey="sn_db",
 926            val=self.sn_db,
 927        )
 928        self.write_to_logfile(
 929            f"initial layer depth stdev (flatness of layer): {self.initial_layer_stdev:.2f}",
 930            mainkey="model_parameters",
 931            subkey="initial_layer_stdev",
 932            val=self.initial_layer_stdev,
 933        )
 934
 935    @staticmethod
 936    def year_plus_fraction() -> str:
 937        # TODO Move this to utils separate module
 938        """
 939        Year Plus Fraction
 940        ----------------------------------------
 941
 942        Method generates a time stamp in the format of
 943        year + fraction of year.
 944
 945        Parameters
 946        ----------
 947        None
 948        
 949        Returns
 950        -------
 951        fraction of the year : str
 952            The time stamp in the format of year + fraction of year
 953
 954        """
 955        now = datetime.datetime.now()
 956        year = now.year
 957        secs_in_year = datetime.timedelta(days=365).total_seconds()
 958        fraction_of_year = (
 959            now - datetime.datetime(year, 1, 1, 0, 0)
 960        ).total_seconds() / secs_in_year
 961        return format(year + fraction_of_year, "14.8f").replace(" ", "")
 962
 963    def hdf_setup(self, hdf_name: str) -> None:
 964        """
 965        Setup HDF files
 966        ---------------
 967
 968        This method sets up the HDF structures
 969
 970        Parameters
 971        ----------
 972        hdf_name : str
 973            The name of the HDF file to be created
 974        
 975        Returns
 976        -------
 977        None
 978        """
 979        num_threads = min(8, mp.cpu_count() - 1)
 980        tables.set_blosc_max_threads(num_threads)
 981        self.hdf_filename = os.path.join(self.temp_folder, hdf_name)
 982        self.filters = tables.Filters(
 983            complevel=5, complib="blosc"
 984        )  # compression with fast write speed
 985        self.h5file = tables.open_file(self.hdf_filename, "w")
 986        self.h5file.create_group("/", "ModelData")
 987
 988    def hdf_init(self, dset_name, shape: tuple, dtype: str = "float64") -> tables.CArray:
 989        
 990        """
 991        HDF Initialize
 992        ----------------------------------------
 993
 994        Method that initializes the HDF chunked
 995        array
 996
 997        Parameters
 998        ----------
 999        dset_name : str
1000            The name of the dataset to be created
1001        shape : tuple
1002
1003        
1004        Returns
1005        -------
1006        new_array: tables.CArray
1007        """
1008        if "float" in dtype:
1009            atom = tables.FloatAtom()
1010        elif "uint8" in dtype:
1011            atom = tables.UInt8Atom()
1012        else:
1013            atom = tables.IntAtom()
1014        group = self.h5file.root.ModelData
1015        new_array = self.h5file.create_carray(
1016            group, dset_name, atom, shape, filters=self.filters
1017        )
1018        return new_array
1019
1020    def hdf_node_list(self):
1021        return [x.name for x in self.h5file.list_nodes("ModelData")]
1022
1023    def hdf_remove_node_list(self, dset_name):
1024        group = self.h5file.root.ModelData
1025        try:
1026            self.h5file.remove_node(group, dset_name)
1027        except:
1028            pass
1029        self.hdf_node_list
1030
1031
1032def triangle_distribution_fix(left, mode, right, random_seed=None):
1033    """
1034    Triangle Distribution Fix
1035    -------------------------
1036
1037    Draw samples from the triangular distribution over the interval [left, right] with modifications.
1038
1039    Ensure some values are drawn at the left and right values by enlarging the interval to
1040    [left - (mode - left), right + (right - mode)]
1041
1042    Parameters
1043    ----------
1044    left: `float`
1045        lower limit
1046    mode: `float`
1047        mode
1048    right: `float`
1049        upper limit
1050    random_seed: `int`
1051        seed to set numpy's random seed
1052
1053    Returns
1054    -------
1055    sn_db: `float`
1056        Drawn samples from parameterised triangular distribution
1057    """
1058    sn_db = 0
1059    while sn_db < left or sn_db > right:
1060        if random_seed:
1061            np.random.seed(random_seed)
1062        sn_db = np.random.triangular(left - (mode - left), mode, right + (right - mode))
1063
1064    return sn_db