Check availability of 30 min raw data

import datetime as dt
import os
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from pyfrhes import read_config
from pyfrhes import get_config_loggers, get_config_logger_files
from pyfrhes import get_config_smartflux_files, read_data

debug = False

# Seaborn's Oranges color palette, start with color white
oranges = mpl.colors.LinearSegmentedColormap.from_list(
    'oranges', sns.color_palette('Oranges'))
ocols = oranges(np.arange(256, dtype=int))
ocols[0] = [1., 1., 1., 1.]  # add white at start
cmap = mpl.colors.ListedColormap(ocols)

ndays = 7
today = dt.datetime.today().date()
# #MC
# today = dt.date(2025, 9, 25)
# #MC
fromday = today - dt.timedelta(days=ndays)
year = today.year

configfile = f'FR-Hes_{year}.cfg'
print(f"Read config file: {configfile}")
config = read_config(configfile)

# loggers
loggers = get_config_loggers(config)
if debug:
    print(f"Loggers: {loggers}")

rfiles = get_config_logger_files(config, loggers, ftype='raw')
if debug:
    rfilenames = { ll: os.path.basename(rfiles[ll]) for ll in rfiles }
    print(f"Raw filenames: {rfilenames}")

dfiles = get_config_logger_files(config, loggers, ftype='DB1')
if debug:
    dfilenames = [ os.path.basename(dfiles[ll]) for ll in dfiles ]
    print(f"DB1 filenames: {dfilenames}")

Read config file: FR-Hes_2025.cfg

Number of missing 30 min raw data

# loggers + smartflux
ndata = np.full((len(loggers) + 1, ndays), 48, dtype=int)
# loggers
for ii, ll in enumerate(loggers):
    if debug:
        print(f"Read file: {rfiles[ll]}")
    df = read_data(rfiles[ll], ftype='raw')
    for dd in range(ndays):
        isday = today - dt.timedelta(days=dd + 1)
        ndata[ii, -dd-1] -= len(df[df.index.date == isday])
        if 'Profile' in ll:
            ndata[ii, -dd-1] += 422 - 48
# smartflux
sfiles = [ os.path.basename(ff)[0:10]
           for ff in get_config_smartflux_files(config, year=year, ftype='raw') ]
for dd in range(ndays):
    isday = today - dt.timedelta(days=dd + 1)
    sisday = isday.strftime('%Y-%m-%d')
    ndata[-1, -dd-1] = 48 - sfiles.count(sisday)
    
loggersmart = loggers.copy()
loggersmart.append('smartflux')
prevdays = []
for dd in range(ndays):
    prevdays.append(today - dt.timedelta(days=ndays - dd))
df = pd.DataFrame(ndata, index=loggersmart,
                  columns=prevdays)

vmax = 48
fig, ax = plt.subplots(figsize=(6.4, df.shape[0]/4.))
sns.heatmap(axes=ax, data=df, vmax=vmax, annot=True, linewidths=0.5,
                cmap=cmap, xticklabels=prevdays,
                yticklabels=df.index, fmt='d')
# ax.set_xlabel('Days before today')
ax.set_ylabel('Logger')
plt.show()