Check availability of 30 min raw data

import datetime as dt
import os
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from pyfrhes import read_config
from pyfrhes import get_config_loggers, get_config_logger_files
from pyfrhes import get_config_smartflux_files, read_data

debug = False

# Seaborn's Oranges color palette, start with color white
oranges = mpl.colors.LinearSegmentedColormap.from_list(
    'oranges', sns.color_palette('Oranges'))
ocols = oranges(np.arange(256, dtype=int))
ocols[0] = [1., 1., 1., 1.]  # add white at start
cmap = mpl.colors.ListedColormap(ocols)

ndays = 7
today = dt.datetime.today().date()
# #MC
# today = dt.date(2025, 9, 25)
# #MC
fromday = today - dt.timedelta(days=ndays)
doy = int(today.strftime('%j'))
if (doy - ndays) < 1:
    year1 = fromday.year
    year = today.year
    configfile = [f'FR-Hes_{year1}.cfg', f'FR-Hes_{year}.cfg']
else:
    year = today.year
    configfile = f'FR-Hes_{year}.cfg'

print(f"Read config file(s): {configfile}")
config = read_config(configfile)

# loggers
loggers = get_config_loggers(config)
if isinstance(loggers[0], list):
    alloggers = []
    for ll in loggers:
        alloggers.extend(ll)
    loggers = list(set(alloggers))
if debug:
    print(f"Loggers: {loggers}")

rfiles = get_config_logger_files(config, loggers, ftype='raw', multiple=True)
if debug:
    if isinstance(rfiles, list):
        for rfils in rfiles:
            rfilenames = { ll: os.path.basename(rfils[ll]) for ll in rfils }
            print(f"Raw filenames: {rfilenames}")
    else:
        rfilenames = { ll: os.path.basename(rfiles[ll]) for ll in rfiles }
        print(f"Raw filenames: {rfilenames}")

dfiles = get_config_logger_files(config, loggers, ftype='DB1')
if debug:
    if isinstance(dfiles, list):
        for dfils in dfiles:
            dfilenames = { ll: os.path.basename(dfils[ll]) for ll in dfils }
            print(f"DB1 filenames: {dfilenames}")
    else:
        dfilenames = { ll: os.path.basename(dfiles[ll]) for ll in dfiles }
        print(f"DB1 filenames: {dfilenames}")

Read config file(s): FR-Hes_2026.cfg

Number of missing 30 min raw data

# loggers + smartflux
ndata = np.full((len(loggers) + 1, ndays), 48, dtype=int)
# loggers
for ii, ll in enumerate(loggers):
    if isinstance(rfiles, list):
        lfiles = [ rr[ll] for rr in rfiles ]
    else:
        lfiles = rfiles[ll]
    if debug:
        print(f"Read file: {lfiles}")
    df = read_data(lfiles, ftype='raw')
    for dd in range(ndays):
        isday = today - dt.timedelta(days=dd + 1)
        ndata[ii, -dd-1] -= len(df[df.index.date == isday])
        if 'Profile' in ll:
            ndata[ii, -dd-1] += 422 - 48
# smartflux
sfiles = get_config_smartflux_files(config, ftype='raw')
if isinstance(sfiles[0], list):
    asfiles = []
    for ll in sfiles:
        asfiles.extend([ os.path.basename(ff)[0:10] for ff in ll ])
else:
    asfiles = [ os.path.basename(ff)[0:10] for ff in sfiles ]
for dd in range(ndays):
    isday = today - dt.timedelta(days=dd + 1)
    sisday = isday.strftime('%Y-%m-%d')
    ndata[-1, -dd-1] = 48 - asfiles.count(sisday)
    
loggersmart = loggers.copy()
loggersmart.append('smartflux')
prevdays = []
for dd in range(ndays):
    prevdays.append(today - dt.timedelta(days=ndays - dd))
df = pd.DataFrame(ndata, index=loggersmart,
                  columns=prevdays)

vmax = 48
fig, ax = plt.subplots(figsize=(6.4, df.shape[0]/4.))
sns.heatmap(axes=ax, data=df, vmax=vmax, annot=True, linewidths=0.5,
                cmap=cmap, xticklabels=prevdays,
                yticklabels=df.index, fmt='d')
# ax.set_xlabel('Days before today')
ax.set_ylabel('Logger')
plt.show()