Skip to content

Update Tutorial Workflow #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
github-actions wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Update Tutorial Workflow #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions metomi/rose/etc/tutorial/cylc-forecasting-workflow/.validate
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.

set -eux
APIKEY="$(head --lines 1 ../api-keys)"
FLOW_NAME="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
cylc lint .
cylc install --workflow-name "$FLOW_NAME" --no-run-name
sed -i "s/DATAPOINT_API_KEY/$APIKEY/" "$HOME/cylc-run/$FLOW_NAME/flow.cylc"
cylc validate --check-circular --icp=2000 "$FLOW_NAME"
cylc play --no-detach --abort-if-any-task-fails "$FLOW_NAME"
cylc clean "$FLOW_NAME"
3 changes: 2 additions & 1 deletion metomi/rose/etc/tutorial/cylc-forecasting-workflow/bin/consolidate-observations
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def plot_wind_data(wind_x, wind_y, x_range, y_range, x_coords, y_coords,
[x[0] for x in z_coords],
[y[1] for y in z_coords],
color='red')
plt.savefig('wind.png')
plt.savefig(f'{os.environ["CYLC_TASK_LOG_DIR"]}/wind.png')


def get_wind_fields():
Expand Down Expand Up @@ -115,4 +115,5 @@ def main():


if __name__ == '__main__':
util.sleep()
main()
4 changes: 3 additions & 1 deletion metomi/rose/etc/tutorial/cylc-forecasting-workflow/bin/forecast
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,11 +150,12 @@ def push_rainfall(rainfall, wind_data, step, resolution, spline_level):
dim_x, dim_y, resolution, resolution,
spline_level)

domain = util.parse_domain(os.environ['DOMAIN'])

while True:
out_of_bounds = []
for itt in range(len(x_values)):
try:
domain = util.parse_domain(os.environ['DOMAIN'])
lng = domain['lng1'] + x_values[itt]
lat = domain['lat1'] + y_values[itt]

Expand Down Expand Up @@ -242,6 +243,7 @@ def main(forecast_interval, forecast_iterations):


if __name__ == '__main__':
util.sleep()
try:
args = [int(sys.argv[1]), int(sys.argv[2])]
except IndexError:
Expand Down
89 changes: 55 additions & 34 deletions metomi/rose/etc/tutorial/cylc-forecasting-workflow/bin/get-observations
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,8 @@ Usage:
get-observations

Environment Variables:
SITE_ID: The four digit DataPoint site code identifying the weather station
we are to fetch results for.
API_KEY: The DataPoint API key, required for getting live weather data.
If un-specified then get-observations will fall back to archive data
from the workflow directory.
SITE_ID: The four digit WMO (World Meteorological Organization)
site code identifying the weather station we are to fetch results for.

"""

Expand All @@ -40,10 +37,8 @@ import re

import requests

import util

BASE_URL = ('http://datapoint.metoffice.gov.uk/public/data/'
'val/wxobs/all/json/{site_id}'
'?res=hourly&time={time}&key={api_key}')

# Compass bearings for ordinate directions.
# NOTE: We measure direction by where the winds have come from!
Expand All @@ -66,22 +61,52 @@ ORDINATES = {
'NNW': '157.5'
}

class Meteorology:
"""
Surface winds tend to be 20-30 degrees backed (vector anticlockwise)
from the winds which steer weather systems:
Friction with the ground surface tends to mean that land surface winds
are as slow as half the speed of the wind at 2000ft. This fudge factor is
a more conservative 1.5:

class NoDataException(Exception):
...
.. seealso::

[Source Book to the Forecaster's Reference Book](https://digital.nmla
.metoffice.gov.uk/IO_011f7cd4-50fc-4903-b556-d24480ea883d/), section
1.2
"""
SURFACE_BACKING = 2
SURFACE_FRICTION = .66
KT_TO_MPH = 1.15078

@staticmethod
def process_direction(direction: str) -> str:
"""Process raw wind direction:

* Convert direction from 10s of degrees to degrees.
* Convert from Oceanographic (wind to) to Meteorological (wind from)
convention.
* Surface Friction Correction
"""
return str(
((int(direction) + 18 - Meteorology.SURFACE_BACKING) % 36) * 10)

@staticmethod
def process_speed(speed: str) -> str:
"""Process Raw wind speed

* Convert to KT to MPH
* Surface Friction Correction
"""
return str(
(
int(speed) * Meteorology.KT_TO_MPH
) / Meteorology.SURFACE_FRICTION
)

def get_datapoint_data(site_id, time, api_key):
"""Get weather data from the DataPoint service."""
# The URL required to get the data.
time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y-%m-%dT%H:%MZ')
url = BASE_URL.format(time=time, site_id=site_id, api_key=api_key)
req = requests.get(url)
if req.status_code != 200:
raise Exception(f'{url} returned exit code {req.status_code}')
# Get the data and parse it as JSON.
print('Opening URL: %s' % url)
return req.json()['SiteRep']['DV']['Location']

class NoDataException(Exception):
...


def get_archived_data(site_id, time):
Expand Down Expand Up @@ -150,11 +175,12 @@ def synop_grab(site_id, time):
raise NoDataException(
f'Request for data failed, raw request was {req.text}')

# Parse the direction from 10's of degrees to degrees:
data['direction'] = str(int(data['direction']) * 10)
# * Parse the direction from 10's of degrees to degrees
# * Convert direction from to direction it's blowing to
data['direction'] = Meteorology.process_direction(data['direction'])

# Convert data in KT to MPH:
data['speed'] = str(int(data['speed']) * 1.15078)
data['speed'] = Meteorology.process_speed(data['speed'])

# Get lat and long from MO Data:
lat, lon = reference_lat_long(site_id)
Expand Down Expand Up @@ -183,7 +209,7 @@ def get_nearby_site(site_id, badsites):
return int(result[0]), dist


def main(site_id, api_key=None):
def main(site_id):
cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']

# Try to get the information from SYNOPS.
Expand All @@ -200,13 +226,8 @@ def main(site_id, api_key=None):
site_id, dist = get_nearby_site(site_id, badsites)

if obs is None:
if api_key:
print('Attempting to get weather data from DataPoint...')
data = get_datapoint_data(site_id, cycle_point, api_key)
else:
print('No API key provided, falling back to archived data')
data = get_archived_data(site_id, cycle_point)

print('Obs unavailable, falling back to archived data')
data = get_archived_data(site_id, cycle_point)
obs = extract_observations(data)

# Write observations.
Expand All @@ -215,5 +236,5 @@ def main(site_id, api_key=None):


if __name__ == '__main__':
main(os.environ['SITE_ID'],
os.environ.get('API_KEY'))
util.sleep()
main(os.environ['SITE_ID'])
Loading