--- /tmp/misterhouse-2.78/bin/get_weather_ca Sat Mar 9 10:54:00 2002 +++ mh/bin/get_weather_ca Mon Mar 10 09:57:57 2003 @@ -17,7 +17,7 @@ # #--------------------------------------------------------------------------- # -# $Id: get_weather_ca,v 1.4 2002/03/09 21:24:20 mhouse Exp $ +# $Id: get_weather_ca,v 1.9 2003/03/10 14:57:29 mhouse Exp $ use strict; @@ -27,14 +27,14 @@ ($Pgm_Name) = $0 =~ /([^.]+)/, $Pgm_Path = '.' unless $Pgm_Name; } -my ($Version) = q$Revision: 1.4 $ =~ /: (\S+)/; # Note: revision number is auto-updated by cvs +my ($Version) = q$Revision: 1.9 $ =~ /: (\S+)/; # Note: revision number is auto-updated by cvs #print "Command: $Pgm_Name @ARGV\n"; #print "Version: $Version\n"; use Getopt::Long; my %parms; -if (!&GetOptions(\%parms, "reget", "h", "help", "v", "city:s", "file=s", "no_log") or +if (!&GetOptions(\%parms, "reget", "h", "help", "v", "debug", "city:s", "file=s", "no_log") or @ARGV or ($parms{h} or $parms{help})) { print< This help text -help => This help text -v => verbose + -debug => debug + -reget => force HTML fetch -city xxx => xxx is the code for the city you want. @@ -83,14 +85,16 @@ use Date::Format; $parms{city} = $config_parms{weather_city} unless $parms{city}; -$parms{city} = 'YYZ' unless $parms{city}; +$parms{city} = 'yyz' unless $parms{city}; -my $WeatherURL = 'http://weatheroffice.ec.gc.ca/scripts/citygen.pl?client=ECCDN_e&city='; -$WeatherURL .= $parms{city}; +my $WeatherURL; +$WeatherURL = sprintf 'http://text.weatheroffice.ec.gc.ca/forecast/city_e.html?%s&b_templatePrint=true', $parms{city}; my $f_weather_html = "$config_parms{data_dir}/web/weather_ca.html"; my $f_weather_data = "$config_parms{data_dir}/weather_data"; +my $debug = 1 if ($parms{debug}); + ############### # get weather # ############### @@ -149,7 +153,7 @@ } if ($response->is_error()) { - printf " %s\n", $response->status_line; + printf "error: %s\n", $response->status_line; } } @@ -165,134 +169,127 @@ %Weather = (); @Weather_Forecast = (); - # we want the tables at 3,0 (data) 2,1 (history) 1,[1,2] (forecast) - # I'd rather use text strings to find them, though. - my $te_data = new HTML::TableExtract(depth => 3, count => 0); - $te_data->parse($html); - - foreach my $ts ($te_data->table_states) { - foreach my $row ($ts->rows) { - $$row[1] =~ s/\s*\n+\s*/ /g; - if ($$row[0] =~ /observed on:\s+(.*)$/i) { - # convert the strangely formatted UTC timestamp to local time. - my $tmp = $1; - $tmp =~ s/\.//g; - $tmp =~ s/\s+at\s+/ /; - $Weather{TimeObserved} = str2time($tmp); - } - elsif ($$row[0] =~ /temp/i) { - $Weather{TempOutdoor} = $$row[1]; - } - elsif ($$row[0] =~ /pressure/i) { - $Weather{Barom} = $$row[1]; - } - elsif ($$row[0] =~ /visibility/i) { - $Weather{Visibility} = $$row[1]; - } - elsif ($$row[0] =~ /humidity/i) { - $Weather{HumidOutdoor} = $$row[1]; - } - elsif ($$row[0] =~ /dewpoint/i) { - $Weather{DewpointOutdoor} = $$row[1]; - } - elsif ($$row[0] =~ /wind/i) { - my $wind = $$row[1]; - $Weather{Wind} = $wind; - $wind =~ /^([A-Z]*)\s*(\d+)\s*km\/h/; - $Weather{WindAvg} = $2; - $Weather{WindAvgDir} = convert_to_degrees($1); - - if ($wind =~ /gusting to (\d+)/) { - $Weather{WindGust} = $1; - $Weather{WindGustDir} = $Weather{WindAvgDir}; - } - else { - $Weather{WindGust} = 0; - $Weather{WindGustDir} = 0; - } - } - else { - if ($$row[0] !~ /^\s*$/m) { - $Weather{Conditions} = $$row[0]; - } - } - } - } + # find the start of the actual data + $html =~ s/.*Current Conditions\s*://m; - my $te_hist = new HTML::TableExtract(depth => 2, count => 1); - $te_hist->parse($html); + $html =~ s///gs; + $html =~ s/<\/strong>//gs; + $html =~ s/\ / /gs; - foreach my $ts ($te_hist->table_states) { - my $normal = 0; - foreach my $row ($ts->rows) { - $$row[1] =~ s/\s*\n+\s*/ /g; - if ($$row[0] =~ /max\s+temp/i) { - if ($normal) { - $Weather{TempMaxNormal} = $$row[1]; - } - else { - $Weather{TempMaxOutdoor} = $$row[1]; - } - } - elsif ($$row[0] =~ /min\s+temp/i) { - if ($normal) { - $Weather{TempMinNormal} = $$row[1]; - } - else { - $Weather{TempMinOutdoor} = $$row[1]; - } - } - elsif ($$row[0] =~ /precip/i) { - $Weather{RainTotal} = $$row[1]; - } - elsif ($$row[0] =~ /normal/i) { - $normal = 1; - } - elsif ($$row[0] =~ /mean\s+temp/i) { - $Weather{TempMeanNormal} = $$row[1]; - } - elsif ($$row[0] =~ /moon\s*rise/i) { - $Weather{Moonrise} = $$row[1]; - } - elsif ($$row[0] =~ /moon\s*set/i) { - $Weather{Moonset} = $$row[1]; - } - } - } +print STDERR $html if ($debug); + + $html =~ m/observed on:*\s+([^<]*)/i; + # convert the strangely formatted UTC timestamp to local time. + my $tmp = $1; + + $tmp =~ s/\.//g; + $tmp =~ s/\s+at\s+/ /; + $Weather{TimeObserved} = str2time($tmp); + + ($html =~ m/temperature\s*:\s*([^<]+)/i) && + ($Weather{TempOutdoor} = $1); + + ($html =~ m/pressure\s*:\s*([^<]+)/i) && + ($Weather{Barom} = $1); + + ($html =~ m/visibility\s*:\s*([^<]+)/i) && + ($Weather{Visibility} = $1); + + ($html =~ m/humidity\s*:\s*([^<]+)/i) && + ($Weather{HumidOutdoor} = $1); + + ($html =~ m/dew\s*point\s*:\s*([^<]+)/i) && + ($Weather{DewpointOutdoor} = $1); + if ($html =~ m/wind speed\s*:\s*([^<]+)/i) { + my $wind = $1; + $wind =~ s/\s*$//; - # The forecast table moves when there is a weather advisory, so we have to - # search for it by hand. We only grab depth 1 tables for efficiency. - my $te_fore = new HTML::TableExtract(depth => 1); - $te_fore->parse($html); - - my @tablestates = ($te_fore->table_states); - my $ts_fore; -table: foreach my $ts (@tablestates) { - foreach my $row ($ts->rows) { - if ($$row[1] =~ /Issued\s+/) { - print "found forecast at ", join(',', $ts->coords), "\n" if $parms{v}; - $ts_fore = $ts; - last table; - } + $Weather{Wind} = $wind; + $wind =~ /^\s*([A-Z]*)\s*(\d+)(
)*\s*km\/h/; + $Weather{WindAvg} = $2; + $Weather{WindAvgDir} = convert_to_degrees($1); + + if ($wind =~ /gusting to (\d+)/) { + $Weather{WindGust} = $1; + $Weather{WindGustDir} = $Weather{WindAvgDir}; + } + else { + $Weather{WindGust} = 0; + $Weather{WindGustDir} = 0; } } - foreach my $row ($ts_fore->rows) { - if ($$row[1] =~ /(Issued.*)$/s) { + ($html =~ m/condition\s*:\s*([^<]+)/i) && + ($Weather{Conditions} = $1); - my $forecast = $1; - $forecast =~ s/\r//g; + my $yesterday = ''; + if ($html =~ m/yesterday :(.*?)<\/ul>/si) { + $yesterday = $1; + } + # new format 20030310 + elsif ($html =~ m;Yesterday\s*(;si) { + $yesterday = $1; + } + print STDERR "Yesterday |$yesterday|\n" if ($debug); + + ($yesterday =~ m/max temp\.*\s*:\s*([^<]+)/i) && + ($Weather{TempMaxOutdoor} = $1); + ($yesterday =~ m/min temp\.*\s*:\s*([^<]+)/i) && + ($Weather{TempMinOutdoor} = $1); + ($yesterday =~ m/precip\. total\s*:\s*([^<]+)/i) && + ($Weather{RainTotal} = $1); + + my $normal = ''; + if ($html =~ m/normal :(.*?)<\/ul>/si) { + $normal = $1; + } + # new format 20030310 + elsif ($html =~ m;normal\s*(;si) { + $normal = $1; + } + print STDERR "Normal |$normal|\n" if ($debug); - my @forecast = split(/\n\n+/m, $forecast); + ($normal =~ m/max temp\.*\s*:\s*([^<]+)/i) && + ($Weather{TempMaxNormal} = $1); + ($normal =~ m/min temp\.*\s*:\s*([^<]+)/i) && + ($Weather{TempMinNormal} = $1); + ($normal =~ m/mean temp\s*:\s*([^<]+)/i) && + ($Weather{TempMeanNormal} = $1); + + my $today = ''; + if ($html =~ m/today :<\/h3>(.*?)<\/ul>/si) { + $today = $1; + } + # new format 20030310 + elsif ($html =~ m;today\s*(;si) { + $today = $1; + } + print STDERR "Today |$today|\n" if ($debug); - map s/\s*\n+\s*/ /g, @forecast; - map s/^\s+//, @forecast; - map s/\s+$//, @forecast; + ($today =~ m/moon\s*rise\s*:\s*([^<]+)/i) && + ($Weather{Moonrise} = $1); + ($today =~ m/moon\s*set\s*:\s*([^<]+)/i) && + ($Weather{Moonset} = $1); - @Weather_Forecast = @forecast; - } +### GET FORECAST +# $html =~ m/]*>(.*?issued.*?)<\/dl>/si; + + my $forecast = ''; + # new format 20030310 + if ($html =~ m;forecast\s*]*>(.*issued.*?);si) { + $forecast = $1; } + print STDERR "Forecast: |$forecast|\n" if ($debug); + + $forecast =~ s/\s*
\s*\n//gs; + $forecast =~ s/\s*<\/dd>\s*/\n/gs; + $forecast =~ s/\s*
\s*//sg; + $forecast =~ s/\s*<\/dt>\s*/\n/gs; + + my @forecast = split(/\n/, $forecast); + + @Weather_Forecast = @forecast; } @@ -306,7 +303,7 @@ foreach $key (keys(%Weather)) { # cleanup the text my $data = $Weather{$key}; - $data =~ s/\s*\xb0\s*[C]*//g; + $data =~ s/\s*(\xb0|°)\s*[C]*//g; $data =~ s/\s*%\s*//g; # $data =~ s;\s*km/h\s*;;g; @@ -320,7 +317,7 @@ foreach $line (@Weather_Forecast) { $perl .= "\n '" . $line . "',"; } - chop($perl); # remove trailing comma + $perl =~ s/,$//; # remove trailing comma $perl .= "\n" . ');' . "\n"; main::file_write($file, $perl);