1
\$\begingroup\$

I'm relatively new to Python have been writing for a few months now. I've started a Pandas Numpy project that starts with parsing large and somewhat sloppy formatted textfile, its not exactly csv but its pretty close.

...
Basetime: 2021102206Z
Forecast Hours:           0hr      1hr... 144hr  
Sfc Prs(mb):         1002.83, 1002.62,...
...
Dry Microburst:         0.00,    0,...
###UA SECTION###
1000mb  GPH (m):          166.88,... 
...
#END LINE 861

create instance from file

  • pd.DataFrame
  • numpy.ndarray
with open('data/2021102206Z.txt', 'r') as f:
    tp = Tarp(f)
    df = tp.asDataFrame()

    print(type(df))
    #<class 'pandas.core.frame.DataFrame'>

    m = tp.asMatrix()
    print(type(m))
    #<class 'numpy.ndarray'>

init

Tarp class parses the applicable data into a Object subclass

list(map(self._series, f)) parse -> reject -> reformat -> setatt


class Tarp(object):

    def __init__(self, f):
        self.props = Object()
        self.models = Object()
        # itterate f as pd.Series
        

    def _series(self, f):
        # split series at : or , or whitespace
        s = pd.Series(f).str.split(r":\s*|,\s*").squeeze()
        # pop key from series to reformat, returns False if key invalid
        k = _pythonic_keys(s.pop(0))
        v = _pythonic_vals(s)
        try:
            if k and v:# k & v = model data
                # force ValueError on non dtype float
                val = np.array(v, dtype=float)
                self.models.keys.append(k)
                setattr(self.models, k, val)
                pass

            elif k:# k not v = station properties
                v = s[0].rstrip()
                if k == "basetime":
                    val = datetime.strptime(v, "%Y%m%d%HZ")
                else:
                    val = v
                self.props.keys.append(k)
                setattr(self.props, k, val)

        except ValueError:# occurs on int array vals

            pass
        except IndexError:# occurs on blank lines

            pass

Methods


    def getmodels(self, key): return getattr(self.models, key)

    def getprops(self, key): return getattr(self.props, key)

    def zip_props(self):
        vals = list(map(self.getprops, self.props.keys))
        return dict(zip(self.props.keys, vals))

    def zip_models(self):
        vals = list(map(self.getmodels, self.models.keys))
        return dict(zip(self.models.keys, vals))

    def asMatrix(self): return np.array(
        list(map(self.getmodels,  self.models.keys)))

    def asDataFrame(self): return pd.DataFrame(self.zip_models())

misc

class Object:
    def __init__(self):
        self.keys = list()
        pass

_wspace = re.compile(r'\s+')
_units = re.compile(r'(?=\s*\(.*)')
_only_wspace = re.compile(r"^\s*$")


def _pythonic_keys(s):
    if bool(_only_wspace.search(s)):
        return False
    else:
        return _wspace.sub("_", _units.split(s)[0]).lower()


def _pythonic_vals(a):
    if len(a) > 0:
        return a[:-1]
    else:
        return False

full code

import numpy as np
import pandas as pd
import re
from datetime import datetime

_wspace = re.compile(r'\s+')
_units = re.compile(r'(?=\s*\(.*)')
_only_wspace = re.compile(r"^\s*$")


def _pythonic_keys(s):
    if bool(_only_wspace.search(s)):
        return False
    else:
        return _wspace.sub("_", _units.split(s)[0]).lower()


def _pythonic_vals(a):
    if len(a) > 0:
        return a[:-1]
    else:
        return False


class Object:
    def __init__(self):
        self.keys = list()
        pass


class Tarp(object):

    def getmodels(self, key): return getattr(self.models, key)

    def getprops(self, key): return getattr(self.props, key)

    def zip_props(self):
        vals = list(map(self.getprops, self.props.keys))
        return dict(zip(self.props.keys, vals))

    def zip_models(self):
        vals = list(map(self.getmodels, self.models.keys))
        return dict(zip(self.models.keys, vals))

    def asMatrix(self): return np.array(
        list(map(self.getmodels,  self.models.keys)))

    def asDataFrame(self): return pd.DataFrame(self.zip_models())

    def __init__(self, f):
        self.props = Object()
        self.models = Object()
        # itterate f as pd.Series
        list(map(self._series, f))


    def _series(self, f):
        # split series at : or , or whitespace
        s = pd.Series(f).str.split(r":\s*|,\s*").squeeze()
        # pop key from series to reformat, returns False if key invalid
        k = _pythonic_keys(s.pop(0))
        v = _pythonic_vals(s)
        try:
            if k and v:
                # force ValueError on non dtype float
                val = np.array(v, dtype=float)
                self.models.keys.append(k)
                setattr(self.models, k, val)
                pass

            elif k:
                v = s[0].rstrip()
                if k == "basetime":
                    val = datetime.strptime(v, "%Y%m%d%HZ")
                else:
                    val = v
                self.props.keys.append(k)
                setattr(self.props, k, val)

        except ValueError:

            pass
        except IndexError:

            pass


def run_test():
    with open('data/2021102206Z.txt', 'r') as f:
        tp = Tarp(f)
        df = tp.asDataFrame()
        print(type(df))
        m = tp.asMatrix()
        print(type(m))

more data

the file is a 144hr forecast by hour for several different values

at line 160 is there a break and the values are upper level values

            
Basetime: 2021102206Z
Spacing: 1/4 Degree 
Region: Worldwide 
StationID: TEST
Station Name: TEST DATA
Latitude:   50.55
Longitude:  -100.84
Elevation: 139.9
Forecast Hours:           0hr      1hr      2hr      3hr      4hr      5hr      6hr      7hr      8hr      9hr     10hr     11hr     12hr     13hr     14hr     15hr     16hr     17hr     18hr     19hr     20hr     21hr     22hr     23hr     24hr     25hr     26hr     27hr     28hr     29hr     30hr     31hr     32hr     33hr     34hr     35hr     36hr     37hr     38hr     39hr     40hr     41hr     42hr     43hr     44hr     45hr     46hr     47hr     48hr     49hr     50hr     51hr     52hr     53hr     54hr     55hr     56hr     57hr     58hr     59hr     60hr     61hr     62hr     63hr     64hr     65hr     66hr     67hr     68hr     69hr     70hr     71hr     72hr     73hr     74hr     75hr     76hr     77hr     78hr     79hr     80hr     81hr     82hr     83hr     84hr     85hr     86hr     87hr     88hr     89hr     90hr     91hr     92hr     93hr     94hr     95hr     96hr     97hr     98hr     99hr    100hr    101hr    102hr    103hr    104hr    105hr    106hr    107hr    108hr    109hr    110hr    111hr    112hr    113hr    114hr    115hr    116hr    117hr    118hr    119hr    120hr    121hr    122hr    123hr    124hr    125hr    126hr    127hr    128hr    129hr    130hr    131hr    132hr    133hr    134hr    135hr    136hr    137hr    138hr    139hr    140hr    141hr    142hr    143hr    144hr
Sfc Prs(mb):         1002.83, 1002.62, 1002.33, 1001.96, 1002.13, 1002.10, 1002.16, 1002.59, 1002.68, 1002.58, 1002.44, 1002.31, 1001.81, 1001.26, 1000.83, 1000.46, 1000.30, 1000.26, 1000.34, 1000.17, 1000.31, 1000.38, 1000.18, 1000.07, 1000.18, 1000.17, 1000.01,  999.85,  999.90, 1000.10, 1000.22, 1000.38, 1000.66, 1000.65, 1000.32,  999.84,  999.16,  998.25,  997.31,  996.85,  996.56,  996.46,  996.74,  996.60,  996.46,  996.32,  995.94,  995.55,  995.17,  994.58,  994.00,  993.41,  993.15,  992.90,  992.64,  992.88,  993.11,  993.34,  992.44,  991.53,  990.63,  989.95,  989.27,  988.59,  988.02,  987.45,  986.88,  986.61,  986.33,  986.06,  985.86,  985.67,  985.48,  985.51,  985.54,  985.56,  986.08,  986.60,  987.12,  987.67,  988.23,  988.79,  988.84,  988.89,  988.95,  989.69,  990.43,  991.18,  992.38,  993.59,  994.79,  995.47,  996.14,  996.81,  997.15,  997.50,  997.84,  997.99,  998.14,  998.30,  998.73,  999.17,  999.61,  999.81, 1000.01, 1000.22,  999.47,  998.73,  997.99,  997.28,  996.57,  995.86,  995.74,  995.63,  995.52,  994.82,  994.11,  993.41,  992.59,  991.77,  990.95,  990.60,  990.26,  989.91,  988.19,  986.47,  984.76,  983.96,  983.17,  982.37,  981.11,  979.84,  978.57,  978.14,  977.71,  977.28,  977.57,  977.85,  978.13,  978.46,  978.79,  979.12,  979.30,  979.49,  979.68,
Mean SLP (mb):       1019.73, 1019.50, 1019.19, 1018.83, 1019.03, 1019.02, 1019.09, 1019.54, 1019.60, 1019.49, 1019.34, 1019.21, 1018.73, 1018.18, 1017.73, 1017.35, 1017.21, 1017.19, 1017.27, 1017.07, 1017.23, 1017.32, 1017.14, 1017.05, 1017.20, 1017.17, 1016.98, 1016.80, 1016.77, 1016.92, 1017.03, 1017.19, 1017.46, 1017.49, 1017.15, 1016.69, 1016.00, 1015.07, 1014.09, 1013.56, 1013.25, 1013.14, 1013.41, 1013.26, 1013.12, 1012.97, 1012.56, 1012.16, 1011.76, 1011.15, 1010.54, 1009.92, 1009.64, 1009.35, 1009.06, 1009.29, 1009.51, 1009.73, 1008.81, 1007.88, 1006.96, 1006.29, 1005.62, 1004.95, 1004.36, 1003.76, 1003.16, 1002.92, 1002.68, 1002.44, 1002.27, 1002.10, 1001.93, 1001.99, 1002.04, 1002.09, 1002.67, 1003.24, 1003.81, 1004.37, 1004.92, 1005.48, 1005.50, 1005.53, 1005.55, 1006.30, 1007.06, 1007.81, 1009.03, 1010.25, 1011.47, 1012.14, 1012.81, 1013.48, 1013.81, 1014.15, 1014.49, 1014.61, 1014.74, 1014.87, 1015.31, 1015.75, 1016.19, 1016.39, 1016.60, 1016.81, 1016.05, 1015.28, 1014.52, 1013.80, 1013.08, 1012.35, 1012.23, 1012.12, 1012.00, 1011.26, 1010.51, 1009.77, 1008.95, 1008.13, 1007.31, 1006.99, 1006.68, 1006.36, 1004.63, 1002.89, 1001.16, 1000.39,  999.61,  998.84,  997.59,  996.35,  995.10,  994.70,  994.30,  993.90,  994.18,  994.47,  994.76,  995.10,  995.44,  995.79,  995.99,  996.20,  996.41,
Altimeter (in. Hg):    30.10,   30.09,   30.09,   30.07,   30.08,   30.08,   30.08,   30.09,   30.10,   30.09,   30.09,   30.08,   30.07,   30.05,   30.04,   30.03,   30.02,   30.02,   30.03,   30.02,   30.02,   30.03,   30.02,   30.02,   30.02,   30.02,   30.02,   30.01,   30.01,   30.02,   30.02,   30.03,   30.04,   30.04,   30.03,   30.01,   29.99,   29.96,   29.93,   29.92,   29.91,   29.91,   29.92,   29.91,   29.91,   29.91,   29.89,   29.88,   29.87,   29.85,   29.84,   29.82,   29.81,   29.80,   29.80,   29.80,   29.81,   29.82,   29.79,   29.76,   29.74,   29.71,   29.69,   29.67,   29.66,   29.64,   29.62,   29.61,   29.61,   29.60,   29.59,   29.59,   29.58,   29.58,   29.58,   29.58,   29.60,   29.61,   29.63,   29.65,   29.66,   29.68,   29.68,   29.68,   29.68,   29.71,   29.73,   29.75,   29.79,   29.82,   29.86,   29.88,   29.90,   29.92,   29.93,   29.94,   29.95,   29.96,   29.96,   29.96,   29.98,   29.99,   30.00,   30.01,   30.02,   30.02,   30.00,   29.98,   29.96,   29.93,   29.91,   29.89,   29.89,   29.88,   29.88,   29.86,   29.84,   29.82,   29.79,   29.77,   29.74,   29.73,   29.72,   29.71,   29.66,   29.61,   29.56,   29.54,   29.51,   29.49,   29.45,   29.41,   29.37,   29.36,   29.35,   29.34,   29.34,   29.35,   29.36,   29.37,   29.38,   29.39,   29.40,   29.40,   29.41,
Press Alt (ft):       285.78,  291.39,  299.50,  309.49,  304.92,  305.71,  304.16,  292.24,  289.64,  292.43,  296.33,  300.04,  313.61,  328.87,  340.80,  350.93,  355.33,  356.43,  354.22,  359.08,  355.09,  353.16,  358.68,  361.81,  358.63,  358.82,  363.37,  367.77,  366.37,  360.80,  357.50,  353.13,  345.54,  345.62,  354.75,  367.99,  386.75,  411.88,  438.08,  450.74,  458.64,  461.38,  453.69,  457.56,  461.43,  465.29,  475.92,  486.54,  497.17,  513.43,  529.69,  545.96,  553.06,  560.16,  567.26,  560.77,  554.29,  547.81,  572.92,  598.05,  623.21,  642.09,  660.98,  679.89,  695.76,  711.65,  727.54,  735.22,  742.90,  750.58,  755.95,  761.32,  766.69,  765.90,  765.11,  764.31,  749.87,  735.44,  721.01,  705.45,  689.90,  674.36,  672.92,  671.48,  670.04,  649.34,  628.65,  607.97,  574.49,  541.04,  507.62,  489.00,  470.38,  451.77,  442.24,  432.72,  423.20,  419.04,  414.88,  410.72,  398.59,  386.46,  374.33,  368.79,  363.24,  357.69,  378.18,  398.68,  419.19,  438.83,  458.49,  478.17,  481.28,  484.40,  487.51,  506.98,  526.46,  545.95,  568.71,  591.48,  614.27,  623.91,  633.55,  643.19,  691.02,  738.91,  786.87,  809.08,  831.29,  853.53,  888.99,  924.49,  960.03,  972.11,  984.19,  996.27,  988.33,  980.39,  972.45,  963.21,  953.97,  944.74,  939.51,  934.28,  929.05,
Density Alt (ft):     -55.22,  -88.18, -135.38, -186.09, -235.12, -262.16, -312.89, -296.80, -142.70,   -0.45,   88.61,  172.51,  274.47,  348.90,  398.48,  378.66,  356.46,  249.88,   90.22,  -10.37, -101.92, -182.53, -205.18, -231.41, -338.80, -394.94, -419.42, -427.96, -418.72, -433.90, -436.02, -374.09, -103.15,  200.48,  426.54,  563.89,  643.07,  731.70,  768.59,  851.75,  836.91,  774.23,  678.68,  684.24,  689.77,  695.11,  705.35,  715.59,  725.83,  742.34,  758.67,  775.21,  850.91,  926.88, 1003.14, 1132.49, 1262.30, 1392.82, 1523.68, 1654.20, 1784.81, 1877.47, 1969.33, 2060.37, 2037.23, 2014.25, 1991.25, 1945.64, 1899.72, 1853.71, 1794.06, 1734.05, 1673.69, 1599.03, 1524.64, 1450.48, 1272.28, 1095.99,  921.63,  829.19,  736.56,  643.31,  710.27,  777.51,  845.04,  795.05,  744.84,  694.83,  596.17,  497.41,  398.98,  341.93,  284.85,  227.93,  189.31,  150.86,  112.19,   91.00,   70.00,   48.99,   26.97,    4.95,  -16.86,   24.73,   66.02,  107.24,  197.02,  286.22,  374.83,  445.25,  515.60,  585.66,  529.25,  472.31,  415.02,  406.81,  398.58,  390.32,  383.68,  377.02,  370.56,  363.20,  355.83,  348.45,  419.28,  490.17,  561.13,  643.70,  726.36,  808.92,  948.00, 1087.20, 1226.51, 1339.39, 1451.95, 1564.38, 1478.83, 1393.31, 1307.80, 1265.79, 1224.16, 1182.28, 1137.12, 1091.92, 1046.88,
2 m agl Tmp (K):      283.50,  283.17,  282.70,  282.18,  281.82,  281.59,  281.20,  281.43,  282.70,  283.86,  284.59,  285.24,  285.94,  286.41,  286.71,  286.48,  286.27,  285.40,  284.10,  283.22,  282.52,  281.89,  281.62,  281.34,  280.52,  280.08,  279.82,  279.69,  279.76,  279.70,  279.71,  280.24,  282.49,  284.96,  286.70,  287.66,  288.10,  288.54,  288.53,  289.02,  288.78,  288.20,  287.45,  287.42,  287.39,  287.36,  287.35,  287.33,  287.32,  287.31,  287.29,  287.28,  287.77,  288.26,  288.75,  289.78,  290.81,  291.85,  292.62,  293.40,  294.18,  294.80,  295.41,  296.03,  295.72,  295.42,  295.12,  294.66,  294.20,  293.74,  293.18,  292.61,  292.05,  291.51,  290.97,  290.44,  289.30,  288.16,  287.02,  286.41,  285.80,  285.19,  285.68,  286.16,  286.64,  286.45,  286.27,  286.08,  285.64,  285.21,  284.77,  284.53,  284.29,  284.05,  283.83,  283.61,  283.39,  283.26,  283.12,  282.99,  282.92,  282.86,  282.80,  283.19,  283.59,  283.99,  284.55,  285.12,  285.68,  286.07,  286.45,  286.84,  286.33,  285.81,  285.30,  285.01,  284.72,  284.44,  284.16,  283.88,  283.60,  283.42,  283.24,  283.06,  283.12,  283.19,  283.25,  283.67,  284.09,  284.51,  285.23,  285.95,  286.68,  287.46,  288.23,  289.01,  288.43,  287.85,  287.27,  287.05,  286.84,  286.62,  286.34,  286.06,  285.78,
2 m agl Temp-D (K):    -4.09,   -4.41,   -4.86,   -5.36,   -5.73,   -5.96,   -6.36,   -6.16,   -4.89,   -3.72,   -2.99,   -2.32,   -1.60,   -1.10,   -0.78,   -0.99,   -1.18,   -2.06,   -3.36,   -4.23,   -4.94,   -5.57,   -5.83,   -6.11,   -6.92,   -7.37,   -7.62,   -7.74,   -7.67,   -7.75,   -7.74,   -7.22,   -4.98,   -2.51,   -0.76,    0.23,    0.71,    1.19,    1.24,    1.76,    1.53,    0.95,    0.19,    0.17,    0.14,    0.12,    0.13,    0.14,    0.14,    0.16,    0.18,    0.20,    0.71,    1.21,    1.71,    2.73,    3.75,    4.77,    5.60,    6.43,    7.26,    7.91,    8.56,    9.21,    8.94,    8.67,    8.40,    7.96,    7.51,    7.07,    6.51,    5.96,    5.41,    4.87,    4.33,    3.79,    2.62,    1.46,    0.29,   -0.35,   -0.99,   -1.63,   -1.15,   -0.67,   -0.19,   -0.42,   -0.65,   -0.88,   -1.38,   -1.88,   -2.38,   -2.66,   -2.94,   -3.22,   -3.46,   -3.69,   -3.93,   -4.07,   -4.22,   -4.36,   -4.45,   -4.53,   -4.62,   -4.23,   -3.85,   -3.46,   -2.86,   -2.25,   -1.65,   -1.22,   -0.80,   -0.37,   -0.88,   -1.39,   -1.90,   -2.15,   -2.39,   -2.64,   -2.88,   -3.11,   -3.34,   -3.50,   -3.67,   -3.83,   -3.67,   -3.51,   -3.35,   -2.89,   -2.42,   -1.96,   -1.17,   -0.37,    0.42,    1.22,    2.02,    2.82,    2.23,    1.63,    1.03,    0.80,    0.57,    0.34,    0.04,   -0.25,   -0.54,
2 m agl Dpt (K):      279.93,  279.97,  279.84,  279.84,  279.86,  279.92,  279.81,  280.17,  280.44,  280.24,  279.96,  280.01,  280.18,  280.19,  280.06,  279.51,  279.16,  278.73,  278.60,  278.48,  278.28,  278.02,  278.45,  278.95,  278.42,  278.22,  278.45,  278.63,  278.90,  278.83,  278.83,  279.16,  279.91,  280.54,  281.34,  282.05,  282.45,  283.00,  283.46,  284.32,  284.71,  284.89,  285.10,  285.45,  285.78,  286.12,  286.01,  285.91,  285.80,  285.62,  285.44,  285.26,  286.00,  286.73,  287.46,  288.58,  289.70,  290.82,  291.36,  291.90,  292.43,  292.36,  292.25,  292.11,  291.65,  291.18,  290.72,  290.58,  290.43,  290.27,  290.19,  290.10,  290.00,  289.28,  288.56,  287.84,  285.82,  283.78,  281.69,  281.59,  281.47,  281.33,  282.44,  283.53,  284.61,  284.41,  284.22,  284.03,  283.56,  283.09,  282.62,  282.14,  281.67,  281.19,  281.13,  281.08,  281.03,  281.07,  281.11,  281.15,  281.23,  281.31,  281.39,  281.43,  281.45,  281.47,  281.14,  280.74,  280.28,  280.28,  280.26,  280.23,  280.30,  280.36,  280.39,  280.62,  280.84,  281.03,  280.88,  280.72,  280.57,  280.84,  281.10,  281.34,  281.71,  282.08,  282.44,  282.87,  283.31,  283.74,  284.50,  285.26,  286.01,  286.48,  286.94,  287.38,  286.83,  286.27,  285.72,  285.31,  284.90,  284.49,  284.02,  283.56,  283.10,
2 m agl WBT (K):      281.58,  281.48,  281.17,  280.96,  280.76,  280.76,  280.45,  280.76,  281.48,  281.94,  282.09,  282.40,  282.81,  283.01,  283.12,  282.71,  282.50,  281.89,  281.27,  280.76,  280.35,  279.94,  280.04,  280.14,  279.48,  279.12,  279.12,  279.12,  279.32,  279.22,  279.27,  279.73,  281.17,  282.60,  283.73,  284.45,  284.86,  285.32,  285.58,  286.19,  286.35,  286.19,  286.04,  286.25,  286.40,  286.60,  286.50,  286.50,  286.40,  286.30,  286.19,  286.09,  286.71,  287.32,  287.94,  289.01,  290.09,  291.17,  291.78,  292.40,  292.96,  293.12,  293.22,  293.32,  292.91,  292.50,  292.14,  291.88,  291.68,  291.42,  291.17,  290.91,  290.70,  290.04,  289.42,  288.76,  287.17,  285.58,  284.04,  283.73,  283.43,  283.07,  283.83,  284.65,  285.48,  285.27,  285.06,  284.86,  284.45,  283.99,  283.53,  283.22,  282.81,  282.50,  282.40,  282.25,  282.09,  282.09,  282.04,  281.99,  281.99,  281.99,  282.04,  282.19,  282.40,  282.60,  282.71,  282.71,  282.76,  282.91,  283.12,  283.22,  283.01,  282.81,  282.60,  282.60,  282.60,  282.60,  282.40,  282.19,  281.99,  281.99,  282.09,  282.09,  282.35,  282.60,  282.81,  283.22,  283.63,  284.04,  284.81,  285.53,  286.30,  286.86,  287.42,  287.99,  287.42,  286.86,  286.30,  285.99,  285.68,  285.37,  284.96,  284.60,  284.25,
Convect. Temp (K):    303.57,  303.13,  303.20,  304.07,  303.98,  303.92,  304.01,  284.02,  283.59,  283.42,  283.80,  285.06,  285.45,  285.68,  286.01,  299.96,  299.77,  300.60,  301.14,  301.41,  301.44,  301.63,  301.08,  300.05,  297.39,  299.94,  300.80,  301.56,  301.54,  301.51,  301.55,  301.72,  302.09,  302.26,  302.30,  302.74,  302.65,  300.64,  299.55,  300.21,  301.11,  302.01,  302.01,  302.06,  302.12,  302.16,  302.32,  302.49,  302.63,  300.58,  300.13,  299.93,  300.10,  299.93,  299.32,  298.63,  297.88,  297.02,  296.63,  296.30,  296.06,  296.56,  296.92,  297.16,  297.80,  299.29,  302.04,  302.33,  302.37,  302.27,  300.75,  298.96,  297.27,  295.20,  293.51,  292.06,  290.58,  289.07,  287.64,  287.07,  286.42,  285.40,  285.87,  286.42,  286.81,  286.53,  286.27,  285.99,  285.79,  285.59,  285.40,  285.15,  284.89,  284.62,  284.51,  284.40,  284.27,  284.27,  284.27,  284.26,  284.12,  284.00,  283.89,  283.84,  283.66,  283.64,  284.14,  320.35,  320.32,  319.86,  319.53,  319.29,  319.07,  318.87,  318.69,  317.98,  317.16,  316.15,  315.87,  315.60,  315.35,  311.88,  309.30,  284.72,  285.32,  284.17,  284.02,  284.41,  284.82,  285.21,  285.91,  286.59,  287.26,  288.94,  289.49,  289.80,  289.48,  289.14,  288.80,  288.46,  288.12,  287.76,  287.43,  287.09,  286.73,
Heat Index (F):        50.62,   50.01,   49.18,   48.24,   47.60,   47.17,   46.47,   46.88,   49.17,   51.27,   52.57,   53.75,   55.01,   55.85,   56.39,   55.97,   55.60,   54.03,   51.69,   50.11,   48.84,   47.71,   47.22,   46.72,   45.26,   44.45,   43.98,   43.75,   43.88,   43.77,   43.80,   44.75,   48.80,   53.25,   56.37,   58.10,   58.89,   59.68,   59.67,   60.55,   60.11,   59.07,   57.72,   57.67,   57.61,   57.56,   57.54,   57.51,   57.49,   57.46,   57.44,   57.42,   58.30,   59.18,   60.06,   61.91,   63.77,   65.63,   67.04,   68.44,   69.84,   70.95,   72.05,   73.69,   73.37,   73.12,   72.95,   72.05,   69.87,   69.04,   68.03,   67.01,   66.00,   65.03,   64.06,   63.10,   61.05,   59.00,   56.95,   55.85,   54.76,   53.66,   54.53,   55.40,   56.26,   55.93,   55.59,   55.25,   54.47,   53.69,   52.90,   52.47,   52.03,   51.59,   51.20,   50.81,   50.41,   50.17,   49.93,   49.69,   49.57,   49.46,   49.35,   50.06,   50.78,   51.49,   52.51,   53.52,   54.53,   55.23,   55.93,   56.62,   55.70,   54.77,   53.85,   53.33,   52.81,   52.29,   51.79,   51.29,   50.79,   50.47,   50.14,   49.81,   49.93,   50.05,   50.16,   50.92,   51.67,   52.43,   53.73,   55.03,   56.33,   57.73,   59.13,   60.53,   59.48,   58.44,   57.39,   57.01,   56.62,   56.23,   55.73,   55.22,   54.71,
Wind Chill (F):        50.62,   50.01,   49.18,   48.24,   47.60,   47.17,   46.47,   46.88,   49.17,   51.27,   52.57,   53.75,   55.01,   55.85,   56.39,   55.97,   55.60,   54.03,   51.69,   50.11,   48.84,   47.71,   47.22,   46.72,   45.26,   44.45,   43.98,   43.75,   43.88,   43.77,   43.80,   44.75,   48.80,   53.25,   56.37,   58.10,   58.89,   59.68,   59.67,   60.55,   60.11,   59.07,   57.72,   57.67,   57.61,   57.56,   57.54,   57.51,   57.49,   57.46,   57.44,   57.42,   58.30,   59.18,   60.06,   61.91,   63.77,   65.63,   67.04,   68.44,   69.84,   70.95,   72.05,   73.16,   72.62,   72.07,   71.53,   70.70,   69.87,   69.04,   68.03,   67.01,   66.00,   65.03,   64.06,   63.10,   61.05,   59.00,   56.95,   55.85,   54.76,   53.66,   54.53,   55.40,   56.26,   55.93,   55.59,   55.25,   54.47,   53.69,   52.90,   52.47,   52.03,   51.59,   51.20,   50.81,   50.41,   50.17,   49.93,   49.69,   49.57,   49.46,   49.35,   50.06,   50.78,   51.49,   52.51,   53.52,   54.53,   55.23,   55.93,   56.62,   55.70,   54.77,   53.85,   53.33,   52.81,   52.29,   51.79,   51.29,   50.79,   50.47,   50.14,   49.81,   49.93,   50.05,   50.16,   50.92,   51.67,   52.43,   53.73,   55.03,   56.33,   57.73,   59.13,   60.53,   59.48,   58.44,   57.39,   57.01,   56.62,   56.23,   55.73,   55.22,   54.71,
FITS (F):              60.81,   60.34,   59.57,   58.79,   58.27,   57.96,   57.30,   57.87,   59.94,   61.55,   62.45,   63.46,   64.61,   65.31,   65.68,   64.98,   64.45,   62.87,   60.85,   59.47,   58.29,   57.19,   57.06,   56.96,   55.41,   54.61,   54.37,   54.30,   54.58,   54.44,   54.46,   55.46,   59.30,   63.38,   66.48,   68.36,   69.27,   70.28,   70.57,   71.85,   71.73,   70.98,   70.00,   70.18,   70.34,   70.52,   70.43,   70.34,   70.25,   70.12,   69.98,   69.85,   71.05,   72.25,   73.44,   75.70,   77.95,   80.21,   81.71,   83.22,   84.72,   85.59,   86.44,   87.26,   86.52,   85.77,   85.02,   84.25,   83.47,   82.68,   81.79,   80.89,   79.98,   78.72,   77.46,   76.20,   73.21,   70.21,   67.18,   66.21,   65.23,   64.23,   65.66,   67.07,   68.48,   68.07,   67.67,   67.27,   66.32,   65.37,   64.43,   63.76,   63.10,   62.43,   62.06,   61.71,   61.35,   61.17,   61.00,   60.82,   60.78,   60.74,   60.69,   61.31,   61.92,   62.52,   63.15,   63.74,   64.28,   64.86,   65.42,   65.98,   65.26,   64.53,   63.78,   63.50,   63.21,   62.91,   62.40,   61.88,   61.37,   61.27,   61.17,   61.05,   61.38,   61.71,   62.04,   62.94,   63.85,   64.75,   66.31,   67.87,   69.43,   70.89,   72.34,   73.78,   72.56,   71.34,   70.12,   69.54,   68.96,   68.38,   67.66,   66.94,   66.23,
2 m agl RH (%):        78.52,   80.53,   82.33,   85.25,   87.44,   89.24,   90.92,   91.78,   85.81,   78.26,   73.18,   70.31,   67.95,   65.96,   64.10,   62.67,   62.01,   63.72,   68.82,   72.37,   74.81,   76.64,   80.45,   84.87,   86.47,   87.96,   90.96,   92.91,   94.26,   94.21,   94.08,   92.82,   83.92,   74.29,   70.00,   69.04,   68.95,   69.56,   71.75,   73.64,   76.72,   80.61,   85.75,   87.91,   90.06,   92.22,   91.68,   91.14,   90.59,   89.61,   88.62,   87.63,   89.11,   90.59,   92.06,   92.62,   93.18,   93.74,   92.40,   91.05,   89.70,   85.99,   82.29,   78.58,   77.76,   76.93,   76.10,   77.56,   79.02,   80.49,   82.95,   85.42,   87.88,   86.82,   85.76,   84.70,   79.88,   75.06,   70.24,   72.57,   74.90,   77.23,   80.66,   84.08,   87.50,   87.45,   87.40,   87.36,   87.11,   86.86,   86.61,   85.24,   83.87,   82.51,   83.42,   84.34,   85.25,   86.28,   87.30,   88.32,   89.19,   90.05,   90.92,   88.76,   86.60,   84.44,   79.51,   74.57,   69.63,   67.86,   66.08,   64.31,   66.85,   69.39,   71.92,   74.47,   77.02,   79.57,   80.21,   80.85,   81.48,   84.02,   86.55,   89.08,   90.95,   92.82,   94.69,   94.81,   94.93,   95.05,   95.29,   95.52,   95.76,   93.86,   91.96,   90.06,   90.16,   90.27,   90.38,   89.22,   88.06,   86.90,   85.84,   84.78,   83.72,
10 m agl Dir:         298.95,  302.50,  308.79,  299.63,  292.80,  286.98,  268.80,  267.70,  268.67,  280.31,  288.14,  292.69,  297.80,  302.88,  311.08,  319.73,  325.41,  345.53,   14.91,   55.37,   85.08,  111.53,  154.23,  180.59,  139.45,   94.70,   99.73,   76.02,   82.79,   98.78,  110.05,  119.49,  125.92,  125.22,  131.40,  135.67,  143.81,  156.04,  149.98,  151.57,  138.75,  125.78,  126.24,  115.52,  106.67,   99.56,  108.58,  117.48,  125.86,  141.16,  155.55,  167.60,  169.99,  172.00,  173.71,  173.77,  173.84,  173.91,  173.34,  172.80,  172.27,  176.46,  180.11,  183.29,  181.13,  179.30,  177.74,  178.17,  178.56,  178.91,  178.99,  179.09,  179.20,  192.90,  211.74,  233.35,  238.35,  243.32,  248.17,  248.51,  248.85,  249.21,  256.61,  264.64,  273.04,  282.95,  290.69,  296.70,  304.82,  313.55,  322.54,  324.25,  326.27,  328.69,  326.94,  325.09,  323.14,  325.79,  328.76,  332.07,  332.09,  332.12,  332.14,  343.91,  357.24,   10.88,   25.49,   47.55,   73.75,   86.61,   98.64,  109.03,  114.34,  119.20,  123.59,  120.95,  119.13,  117.81,  119.46,  121.32,  123.43,  124.21,  125.26,  126.74,  130.31,  132.09,  133.15,  135.12,  137.25,  139.57,  143.71,  148.38,  153.60,  176.75,  205.49,  228.70,  232.07,  234.95,  237.43,  249.01,  259.06,  267.38,  268.20,  269.05,  269.92,
10 m agl Spd (kt):      4.00,    4.00,    4.00,    4.00,    4.00,    4.00,    4.00,    4.00,    5.00,    7.00,    7.00,    7.00,    7.00,    7.00,    7.00,    6.00,    5.00,    4.00,    4.00,    3.00,    2.00,    3.00,    3.00,    4.00,    1.00,    3.00,    3.00,    3.00,    3.00,    4.00,    4.00,    4.00,    6.00,    6.00,    6.00,    6.00,    6.00,    6.00,    6.00,    7.00,    5.00,    5.00,    5.00,    5.00,    5.00,    6.00,    6.00,    6.00,    6.00,    6.00,    7.00,    8.00,    8.00,    9.00,   10.00,   10.00,   10.00,   10.00,   10.00,   10.00,   10.00,   11.00,   12.00,   13.00,   14.00,   15.00,   16.00,   17.00,   18.00,   19.00,   18.00,   17.00,   15.00,   13.00,   11.00,   11.00,   11.00,   11.00,   11.00,   11.00,   11.00,   11.00,   10.00,   10.00,   10.00,   11.00,   13.00,   14.00,   14.00,   13.00,   13.00,   12.00,   11.00,   10.00,   10.00,   10.00,    9.00,    9.00,    8.00,    8.00,    7.00,    7.00,    6.00,    6.00,    6.00,    6.00,    5.00,    4.00,    4.00,    4.00,    4.00,    5.00,    5.00,    5.00,    5.00,    6.00,    8.00,    9.00,    8.00,    8.00,    7.00,    6.00,    6.00,    5.00,    7.00,    9.00,   12.00,   11.00,   11.00,   10.00,   10.00,    9.00,    9.00,    7.00,    7.00,    9.00,    9.00,   10.00,   11.00,   11.00,   12.00,   14.00,   14.00,   13.00,   13.00,
###....line 160UA SECTION###
1000mb  GPH (m):          166.88,  165.16,  162.34,  159.50,  161.16,  160.70,  161.16,  164.98,  165.70,  164.82,  163.34,  162.34,  158.34,  153.52,  150.18,  147.18,  145.72,  145.72,  145.84,  144.54,  146.00,  146.76,  144.64,  144.12,  144.90,  144.54,  143.18,  141.84,  142.66,  143.94,  144.94,  146.66,  148.78,  148.96,  145.84,  141.84,  136.12,  128.66,  120.44,  116.66,  114.50,  113.42,  116.02,  114.77,  113.53,  112.28,  109.00,  105.72,  102.44,   97.50,   92.56,   87.62,   85.30,   82.99,   80.68,   82.66,   84.64,   86.62,   78.45,   70.28,   62.12,   55.95,   49.78,   43.62,   38.65,   33.68,   28.71,   26.56,   24.41,   22.25,   20.78,   19.30,   17.82,   18.34,   18.86,   19.38,   24.23,   29.07,   33.92,   39.10,   44.28,   49.46,   49.72,   49.99,   50.26,   56.58,   62.90,   69.22,   79.36,   89.50,   99.64,  105.31,  110.98,  116.64,  119.50,  122.35,  125.20,  126.42,  127.64,  128.86,  132.69,  136.51,  140.34,  142.01,  143.67,  145.34,  139.01,  132.67,  126.34,  120.35,  114.37,  108.38,  107.47,  106.57,  105.66,   99.96,   94.27,   88.58,   81.61,   74.65,   67.68,   64.92,   62.16,   59.39,   44.82,   30.25,   15.68,    8.73,    1.77,   -5.18,  -16.35,  -27.52,  -38.68,  -42.91,  -47.13,  -51.36,  -48.50,  -45.64,  -42.78,  -39.97,  -37.16,  -34.34,  -32.48,  -30.62,  -28.76,
1000mb  GPH DVal(m):       56.00,   54.28,   51.46,   48.62,   50.28,   49.82,   50.28,   54.09,   54.82,   53.93,   52.46,   51.46,   47.46,   42.64,   39.30,   36.30,   34.84,   34.84,   34.96,   33.66,   35.12,   35.87,   33.75,   33.23,   34.02,   33.66,   32.30,   30.96,   31.77,   33.05,   34.05,   35.77,   37.89,   38.07,   34.96,   30.96,   25.23,   17.77,    9.55,    5.77,    3.61,    2.53,    5.14,    3.89,    2.64,    1.40,   -1.88,   -5.17,   -8.45,  -13.39,  -18.33,  -23.27,  -25.58,  -27.89,  -30.21,  -28.23,  -26.25,  -24.27,  -32.43,  -40.60,  -48.77,  -54.94,  -61.10,  -67.27,  -72.24,  -77.20,  -82.17,  -84.33,  -86.48,  -88.63,  -90.11,  -91.59,  -93.07,  -92.55,  -92.02,  -91.50,  -86.66,  -81.81,  -76.97,  -71.79,  -66.61,  -61.43,  -61.16,  -60.89,  -60.63,  -54.30,  -47.98,  -41.66,  -31.52,  -21.38,  -11.24,   -5.58,    0.09,    5.76,    8.61,   11.46,   14.32,   15.54,   16.76,   17.98,   21.80,   25.63,   29.46,   31.12,   32.79,   34.46,   28.12,   21.79,   15.46,    9.47,    3.48,   -2.50,   -3.41,   -4.32,   -5.23,  -10.92,  -16.61,  -22.31,  -29.27,  -36.24,  -43.20,  -45.97,  -48.73,  -51.49,  -66.06,  -80.63,  -95.20, -102.16, -109.11, -116.07, -127.23, -138.40, -149.57, -153.79, -158.02, -162.25, -159.39, -156.53, -153.67, -150.85, -148.04, -145.23, -143.37, -141.51, -139.65,
1000mb  Temp (K):         283.88,  283.48,  283.09,  282.57,  282.16,  281.83,  281.51,  281.33,  282.10,  282.98,  283.56,  284.11,  284.78,  285.36,  285.80,  285.93,  285.93,  285.70,  284.88,  284.21,  283.76,  283.39,  282.76,  282.08,  281.82,  281.54,  281.03,  280.67,  280.38,  280.28,  280.23,  280.44,  282.13,  284.26,  285.90,  286.92,  287.64,  288.15,  288.42,  288.76,  288.83,  288.58,  288.07,  287.99,  287.90,  287.81,  287.87,  287.93,  287.98,  288.04,  288.09,  288.15,  288.63,  289.10,  289.58,  290.41,  291.24,  292.07,  292.93,  293.79,  294.66,  295.37,  296.08,  296.79,  296.65,  296.51,  296.37,  295.86,  295.34,  294.83,  294.26,  293.69,  293.13,  292.67,  292.20,  291.74,  290.46,  289.18,  287.91,  287.13,  286.35,  285.57,  286.00,  286.42,  286.84,  286.70,  286.56,  286.43,  285.97,  285.52,  285.07,  284.78,  284.49,  284.20,  283.95,  283.70,  283.45,  283.31,  283.17,  283.03,  282.94,  282.84,  282.75,  283.01,  283.27,  283.53,  284.01,  284.50,  284.98,  285.48,  285.98,  286.48,  286.25,  286.02,  285.79,  285.50,  285.22,  284.93,  284.73,  284.53,  284.34,  284.12,  283.91,  283.70,  283.85,  284.01,  284.16,  284.62,  285.07,  285.52,  286.27,  287.02,  287.77,  288.59,  289.41,  290.24,  289.73,  289.22,  288.72,  288.49,  288.27,  288.05,  287.72,  287.40,  287.07,
1000mb  Temp DVal(K):      -3.56,   -3.96,   -4.35,   -4.87,   -5.28,   -5.61,   -5.93,   -6.11,   -5.34,   -4.46,   -3.88,   -3.33,   -2.66,   -2.08,   -1.64,   -1.51,   -1.51,   -1.74,   -2.56,   -3.23,   -3.68,   -4.05,   -4.68,   -5.36,   -5.62,   -5.90,   -6.41,   -6.77,   -7.06,   -7.15,   -7.21,   -7.00,   -5.31,   -3.17,   -1.53,   -0.51,    0.20,    0.71,    0.98,    1.32,    1.39,    1.14,    0.63,    0.55,    0.46,    0.37,    0.43,    0.49,    0.55,    0.60,    0.65,    0.71,    1.19,    1.67,    2.14,    2.97,    3.80,    4.63,    5.49,    6.35,    7.22,    7.93,    8.64,    9.36,    9.21,    9.07,    8.93,    8.42,    7.90,    7.39,    6.82,    6.26,    5.69,    5.23,    4.76,    4.30,    3.02,    1.74,    0.47,   -0.31,   -1.09,   -1.87,   -1.44,   -1.02,   -0.60,   -0.74,   -0.88,   -1.01,   -1.47,   -1.92,   -2.37,   -2.66,   -2.95,   -3.24,   -3.49,   -3.74,   -3.99,   -4.13,   -4.27,   -4.41,   -4.50,   -4.59,   -4.68,   -4.43,   -4.17,   -3.91,   -3.43,   -2.94,   -2.45,   -1.96,   -1.46,   -0.96,   -1.19,   -1.42,   -1.65,   -1.94,   -2.22,   -2.51,   -2.71,   -2.91,   -3.10,   -3.32,   -3.53,   -3.74,   -3.59,   -3.43,   -3.28,   -2.82,   -2.37,   -1.92,   -1.17,   -0.42,    0.33,    1.15,    1.97,    2.80,    2.29,    1.78,    1.28,    1.06,    0.83,    0.61,    0.28,   -0.04,   -0.37,
Improve this question
\$\endgroup\$
2
  • 1
    \$\begingroup\$ Is all your Python in one file? Could you format it as such here? Could you give a few more lines of your text file, so that we could copy and paste it into our own copy of the text file to run your script? \$\endgroup\$
    – Teepeemm
    Oct 26, 2021 at 17:58
  • \$\begingroup\$ I updated the code with the full context, you should be able to save that text file in a folder called data/2021102206Z.txt and call run_test() \$\endgroup\$
    – Jason Leaver
    Oct 26, 2021 at 18:19

1 Answer 1

Reset to default
0
\$\begingroup\$
  • pass is for when Python requires you to have something, but there's nothing else to have. So it works in the except clauses, but otherwise isn't necessary.
  • But catching an exception without doing anything else is usually a bad idea. You want to know that something happened, especially if you're dealing with sloppy data. The only occurrence I ran into was an IndexError on the ### line. I added a conditional to avoid that, and removed the except clauses.
  • Classes automatically inherit from object; you don't need to do so explicitly.
  • Don't use list(map) instead of a for loop in Tarp.__init__. Generally, comprehensions are better than map.
  • Having _pythonic_keys sometimes return a str and sometimes False is a bit odd. Since you're testing its truthiness, lets return None instead of False.
  • But actually, it turns out that _pythonic_key doesn't need its if check. If the string is only whitespace, then (formerly) v ended up empty which caused an IndexError and nothing happened. Now, key ends up empty for a quick return.
  • I find it much easier to read a simple regular expression than to remember what it's doing from the variable name. One could argue that compiling them gives a bit of efficiency, but (1) that's way too premature of an optimization and (2) Python caches its regexs, so it doesn't end up being an optimization anyway.
  • Object is a terrible name for a class. Maybe KeysList? But for that matter, the only point of the props and models property is for the key/value object that it's keeping. Python already does that with dicts, so lets use dicts instead. (If you really needed a key/value object, a Named Tuple could do the job.)
  • Now we realize that the zip functions are really just returning the dicts, so lets do that instead. Also, you never used zip_props. And asMatrix is just using the dict values. This eliminates the need for getmodels and getprops.
  • You ended up with forecast_hours in props with a value that was everything else. I assume that was a bug, and moved it to models.
  • Single letter variables are ok for a short time and if their meaning is clear. k and v (kind of) pass this test, but s and f don't.
  • Overwriting v makes it a bit harder to debug. But val doesn't gain you much, so I repurposed val to be the new v (and got rid of the old val).
  • _pythonic_vals does not behave like _pythonic_keys. It seems that the main purpose is to remove the final entry. But it shouldn't unilaterally do that without looking at that final entry.
  • _series takes a line of text, constructs a length 1 series, and then splits it. Don't go through pandas just to split. I had a hard time understanding this method in general, as it had several of these items that I changed.
  • In a similar spirit of avoiding unnecessary methods, I would avoid using regex when you can do the same thing with string methods (eg, _only_wspace and _units).
  • Your comment says that _series splits on : or , or whitespace, but it's really : or , and strips whitespace after that. I would consider that statement too obvious to comment, and you definitely don't want a comment to be in conflict with the code.
  • _pythonic_key is only called from within Tarp, so I moved it there. I also made it a staticmethod.
  • I added type hints, so that you can run python -m mypy thisfile.py.
  • I added a __main__ guard, so that this could end up in its own package. I also added a few more print statements to run_test().
  • You switch between " and '. If there's a reason for that, that would be fine. If not, you should be more consistent.
import numpy as np # type: ignore
import pandas as pd # type: ignore
import re
from datetime import datetime
from typing import Dict,List,TextIO,Union

class Tarp:
    def __init__(self, filein: TextIO):
        self.props: Dict[str,Union[str,datetime]] = {}
        self.models: Dict[str,np.array] = {}
        for line in filein:
            self._series(line)
    @staticmethod
    def _pythonic_key(s:str) -> str:
        return re.sub(r'\s+','_',s.split('(',1)[0].rstrip()).lower()
    def _series(self, line: str) -> None:
        rowentries: List[str] = re.split(r':\s*|,\s*',line)
        key: str = self._pythonic_key(rowentries.pop(0))
        if rowentries:
            if not re.search(r'\S',rowentries[-1]):
                rowentries.pop()
        if not key or not rowentries:
            return
        if len(rowentries)>1:
            self.models[key] = np.array(rowentries, dtype=float)
        else:
            val: str = rowentries[0].rstrip()
            if key == 'basetime':
                self.props[key] = datetime.strptime(val, '%Y%m%d%HZ')
            elif key == 'forecast_hours':
                self.models[key] = np.array( [ t.rstrip('hr') for t in val.split() ] ,
                                            dtype=int)
            else:
                self.props[key] = val
    def asMatrix(self) -> np.array:
        return np.array( self.models.values() )
    def asDataFrame(self) -> pd.DataFrame:
        return pd.DataFrame(self.models)

def run_test():
    with open('data/2021102206Z.txt', 'r') as filein:
        tp = Tarp(filein)
        print('props:',tp.props)
        df = tp.asDataFrame()
        print(df.shape)
        print(df.dtypes)
        print(type(df))
        print(df)
        m = tp.asMatrix()
        print(type(m))

if __name__ == '__main__':
    run_test()
Improve this answer
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge that you have read and understand our privacy policy and code of conduct.

Not the answer you're looking for? Browse other questions tagged or ask your own question.