[freeside-commits] branch FREESIDE_4_BRANCH updated. bd1a73ccafba5c67e449aa92a88d70c96b031dab
Mitch Jackson
mitch at freeside.biz
Sun Dec 23 14:50:16 PST 2018
The branch, FREESIDE_4_BRANCH has been updated
via bd1a73ccafba5c67e449aa92a88d70c96b031dab (commit)
via 3f13f3395334c560215fdd1d95e44418ac1144dc (commit)
via 4b5e294d32ba1a2834f44f2dbcd42a276a0a75f1 (commit)
via 2189e5b3fcfc2e9dbba69824bedced93ada56991 (commit)
via 68433955458415f8a1c48629e736e6d4bcf7a985 (commit)
via 584e97b0dc65ac774900fc848e308e540c8dd1a6 (commit)
via 09ab7084366594c1fd76eae1375aca75db774ba8 (commit)
from 06d26720a9c77386bc7353bb9c501a2d46007f83 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit bd1a73ccafba5c67e449aa92a88d70c96b031dab
Author: Mitch Jackson <mitch at freeside.biz>
Date: Sun Dec 23 16:40:26 2018 -0500
RT# 80488 Correct typo
diff --git a/bin/wa_tax_rate_update b/bin/wa_tax_rate_update
index cb5814537..ad14687c9 100755
--- a/bin/wa_tax_rate_update
+++ b/bin/wa_tax_rate_update
@@ -139,7 +139,7 @@ sub HELP_MESSAGE {
defaults as conf value 'tax_district_taxname'
-y year Year for data file download
-q quarter Quarter of data file to download
- -t lookup Try to fix cust_location records without a district
+ -l lookup Try to fix cust_location records without a district
";
exit;
commit 3f13f3395334c560215fdd1d95e44418ac1144dc
Author: Mitch Jackson <mitch at freeside.biz>
Date: Sun Dec 23 16:37:20 2018 -0500
RT# 80488 Add dependency libspreadsheet-parsexlsx-perl
diff --git a/debian/control b/debian/control
index f1f87af31..963b814e8 100644
--- a/debian/control
+++ b/debian/control
@@ -101,7 +101,8 @@ Depends: aspell-en,gnupg,ghostscript,gsfonts,gzip,latex-xcolor,
libmap-splat-perl, libdatetime-format-ical-perl, librest-client-perl,
libgeo-streetaddress-us-perl, libbusiness-onlinepayment-perl,
libnet-vitelity-perl (>= 0.05), libnet-sslglue-perl, libexpect-perl,
- libspreadsheet-parsexlsx-perl, libunicode-truncate-perl (>= 0.303-1)
+ libspreadsheet-parsexlsx-perl, libunicode-truncate-perl (>= 0.303-1),
+ libspreadsheet-xlsx-perl
Conflicts: libparams-classify-perl (>= 0.013-6)
Replaces: freeside (<<4)
Breaks: freeside (<<4)
commit 4b5e294d32ba1a2834f44f2dbcd42a276a0a75f1
Author: Mitch Jackson <mitch at freeside.biz>
Date: Sun Dec 23 15:36:51 2018 -0500
RT# 80488 Prevent rollback of system log messages
diff --git a/FS/FS/Cron/tax_rate_update.pm b/FS/FS/Cron/tax_rate_update.pm
index b0745e409..72ca14535 100755
--- a/FS/FS/Cron/tax_rate_update.pm
+++ b/FS/FS/Cron/tax_rate_update.pm
@@ -117,10 +117,14 @@ sub wa_sales {
=head2 wa_sales_log_customer_without_tax_district
-For any active customers with cust_location records in WA state,
-if a cust_location record has no tax district, find the correct
-district using WA DOR API, or if not possible, generate an error
-message into system log so address can be corrected
+For any cust_location records
+* In WA state
+* Attached to non cancelled packages
+* With no tax district
+
+Classify the tax district for the record using the WA State Dept of
+Revenue API. If this fails, generate an error into system log so
+address can be corrected
=cut
@@ -144,12 +148,31 @@ sub wa_sales_log_customer_without_tax_district {
state => 'WA',
district => undef,
},
- addl_from => 'LEFT JOIN cust_main USING (custnum)',
- extra_sql => sprintf 'AND ( %s ) ', FS::cust_main->active_sql,
+ addl_from => '
+ LEFT JOIN cust_main USING (custnum)
+ LEFT JOIN cust_pkg ON cust_location.locationnum = cust_pkg.locationnum
+ ',
+ extra_sql => sprintf(
+ '
+ AND cust_pkg.pkgnum IS NOT NULL
+ AND (
+ cust_pkg.cancel > %s
+ OR cust_pkg.cancel IS NULL
+ )
+ ', time()
+ ),
);
for my $cust_location ( qsearch( \%qsearch_cust_location )) {
local $@;
+ log_info_and_warn(
+ sprintf
+ 'Attempting to classify district for cust_location ' .
+ 'locationnum(%s) address(%s)',
+ $cust_location->locationnum,
+ $cust_location->address1,
+ );
+
eval {
FS::geocode_Mixin::process_district_update(
'FS::cust_location',
@@ -158,16 +181,13 @@ sub wa_sales_log_customer_without_tax_district {
};
if ( $@ ) {
+ # Error indicates a crash, not an error looking up district
+ # process_district_udpate will generate log messages for those errors
log_error_and_warn(
- sprintf "Failed to classify district for cust_location(%s): %s",
+ sprintf "Classify district error for cust_location(%s): %s",
$cust_location->locationnum,
$@
);
- } else {
- log_info_and_warn(
- sprintf "Classified district for cust_location(%s)",
- $cust_location->locationnum
- );
}
sleep 1; # Be polite to WA DOR API
@@ -289,8 +309,6 @@ sub wa_sales_update_tax_table {
Create or update the L<FS::cust_main_county> records with new data
-
-
=cut
sub wa_sales_update_cust_main_county {
@@ -337,7 +355,9 @@ sub wa_sales_update_cust_main_county {
if (
$row->tax == ( $district->{tax_combined} * 100 )
- && $row->taxname eq $args->{taxname}
+ && $row->taxname eq $args->{taxname}
+ && uc $row->county eq uc $district->{county}
+ && uc $row->city eq uc $district->{city}
) {
$same_count++;
next;
diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm
index 96ce0764b..2e44364f2 100644
--- a/FS/FS/Misc/Geo.pm
+++ b/FS/FS/Misc/Geo.pm
@@ -147,7 +147,7 @@ sub get_district {
Expects output of location_hash() as parameter
-Dies on error, or if tax rate cannot be found using given address
+Returns undef on error, or if tax rate cannot be found using given address
Query the WA State Dept of Revenue API with an address, and return
tax district information for that address.
@@ -172,12 +172,23 @@ Returns a hashref with the following keys:
- country US
- exempt_amount 0
+If api returns no district for address, generates system log error
+and returns undef
+
=cut
sub wa_sales {
+
+ #
+ # no die():
+ # freeside-queued will issue dbh->rollback on die() ... this will
+ # also roll back system log messages about errors :/ freeside-queued
+ # doesn't propgate die messages into the system log.
+ #
+
my $location_hash = shift;
- # Return without die() when called with pointless context
+ # Return when called with pointless context
return
unless $location_hash
&& ref $location_hash
@@ -227,8 +238,10 @@ sub wa_sales {
my $error =
sprintf "Problem parsing XML from API URL(%s): %s",
$prepared_url, $@;
+
$log->error( $error );
- die $error;
+ warn $error;
+ return;
}
my ($res_root) = $dom->findnodes('/response');
@@ -255,8 +268,10 @@ sub wa_sales {
$res_code ? $api_response_codes[$res_code] : 'n/a',
$location_hash->{address1},
$prepared_url;
+
$log->error( $error );
- die "$error\n";
+ warn "$error\n";
+ return;
}
my %response = (
diff --git a/bin/wa_tax_rate_update b/bin/wa_tax_rate_update
old mode 100644
new mode 100755
index fef126d34..cb5814537
--- a/bin/wa_tax_rate_update
+++ b/bin/wa_tax_rate_update
@@ -131,7 +131,7 @@ sub HELP_MESSAGE {
Tool to update city/district sales tax rates in I<cust_main_county> from
the Washington State Department of Revenue website.
- Usage: [-f filename] [-t taxname] [-y year] [-q quarter] [-l] freeside_username
+ Usage: wa_tax_rate_update [-f filename] [-t taxname] [-y year] [-q quarter] [-l] freeside_username
Optional Options:
-f filename Skip download, and process the specified filename
commit 2189e5b3fcfc2e9dbba69824bedced93ada56991
Author: Mitch Jackson <mitch at freeside.biz>
Date: Sat Dec 22 03:27:22 2018 -0500
RT# 80488 WA tax tables maintained with Cron
* Some re-work for WA tax tables when using tax classes
* Relocated functions from wa_tax_rate_update utility script
into FS::Cron::wa_tax_rate_update library
* Cron downloads tax database monthly instead of daily
* Cron populates entire WA tax table, instead of piecemeal
for existing customer locations
* Cron will attempt to classify cust_location in WA without
a determined district, and generate system log errors
upon failure
diff --git a/FS/FS/Cron/tax_rate_update.pm b/FS/FS/Cron/tax_rate_update.pm
index fec696fbb..b0745e409 100755
--- a/FS/FS/Cron/tax_rate_update.pm
+++ b/FS/FS/Cron/tax_rate_update.pm
@@ -9,106 +9,593 @@ FS::Cron::tax_rate_update
Cron routine to update city/district sales tax rates in I<cust_main_county>.
Currently supports sales tax in the state of Washington.
+=head2 wa_sales
+
+=item Tax Rate Download
+
+Once each month, update the tax tables from the WA DOR website.
+
+=item Customer Address Rate Classification
+
+Find cust_location rows in WA with no tax district. Try to determine
+a tax district. Otherwise, generate a log error that address needs
+to be correctd.
+
=cut
use strict;
use warnings;
-use FS::Conf;
-use FS::Record qw(qsearch qsearchs dbh);
-use FS::cust_main_county;
-use FS::part_pkg_taxclass;
+use feature 'state';
+
+use Exporter;
+our @EXPORT_OK = qw(
+ tax_rate_update
+ wa_sales_update_tax_table
+ wa_sales_log_customer_without_tax_district
+);
+
+use Carp qw(croak);
use DateTime;
-use LWP::UserAgent;
use File::Temp 'tempdir';
use File::Slurp qw(read_file write_file);
+use LWP::UserAgent;
+use Spreadsheet::XLSX;
use Text::CSV;
-use Exporter;
-our @EXPORT_OK = qw(tax_rate_update);
+use FS::Conf;
+use FS::cust_main;
+use FS::cust_main_county;
+use FS::geocode_Mixin;
+use FS::Log;
+use FS::part_pkg_taxclass;
+use FS::Record qw(qsearch qsearchs dbh);
+use FS::upgrade_journal;
+
our $DEBUG = 0;
+=head1 FUNCTIONS
+
+=head2 tax_rate_update
+
+Cron routine for freeside_daily.
+
+Run one of the available cron functions based on conf value tax_district_method
+
+=cut
+
sub tax_rate_update {
- my %opt = @_;
- my $oldAutoCommit = $FS::UID::AutoCommit;
- local $FS::UID::AutoCommit = 0;
- my $dbh = dbh;
-
- my $conf = FS::Conf->new;
- my $method = $conf->config('tax_district_method');
- return if !$method;
-
- my $taxname = $conf->config('tax_district_taxname') || '';
-
- FS::cust_main_county->lock_table;
- if ($method eq 'wa_sales') {
- # download the update file
- my $now = DateTime->now;
- my $yr = $now->year;
- my $qt = $now->quarter;
- my $file = "Rates${yr}Q${qt}.zip";
- my $url = 'http://dor.wa.gov/downloads/Add_Data/'.$file;
- my $dir = tempdir();
- chdir($dir);
- my $ua = LWP::UserAgent->new;
- warn "Downloading $url...\n" if $DEBUG;
- my $response = $ua->get($url);
- if ( ! $response->is_success ) {
- die $response->status_line;
- }
- write_file($file, $response->decoded_content);
+ # Currently only wa_sales is supported
+ my $tax_district_method = conf_tax_district_method();
+
+ return unless $tax_district_method;
+
+ if ( exists &{$tax_district_method} ) {
+ my $func = \&{$tax_district_method};
+ $func->();
+ } else {
+ my $log = FS::Log->new('tax_rate_update');
+ $log->error( "Unhandled tax_district_method($tax_district_method)" );
+ }
+
+}
+
+=head2 wa_sales
+
+Monthly: Update the complete WA state tax tables
+Every Run: Log errors for cust_location records without a district
+
+=cut
+
+sub wa_sales {
+
+ return
+ unless conf_tax_district_method()
+ && conf_tax_district_method() eq 'wa_sales';
+
+ my $dt_now = DateTime->now;
+ my $year = $dt_now->year;
+ my $quarter = $dt_now->quarter;
+
+ my $journal_label =
+ sprintf 'wa_sales_update_tax_table_%sQ%s', $year, $quarter;
+
+ unless ( FS::upgrade_journal->is_done( $journal_label ) ) {
+ local $@;
+
+ eval{ wa_sales_update_tax_table(); };
+ log_error_and_die( "Error updating tax tables: $@" )
+ if $@;
+ FS::upgrade_journal->set_done( $journal_label );
+ }
+
+ wa_sales_log_customer_without_tax_district();
+
+ '';
+
+}
- # parse it
- system('unzip', $file);
- $file =~ s/\.zip$/.csv/;
- if (! -f $file) {
- die "$file not found in zip archive.\n";
+=head2 wa_sales_log_customer_without_tax_district
+
+For any active customers with cust_location records in WA state,
+if a cust_location record has no tax district, find the correct
+district using WA DOR API, or if not possible, generate an error
+message into system log so address can be corrected
+
+=cut
+
+sub wa_sales_log_customer_without_tax_district {
+
+ return
+ unless conf_tax_district_method()
+ && conf_tax_district_method() eq 'wa_sales';
+
+ my %qsearch_cust_location = (
+ table => 'cust_location',
+ select => '
+ cust_location.locationnum,
+ cust_location.custnum,
+ cust_location.address1,
+ cust_location.city,
+ cust_location.state,
+ cust_location.zip
+ ',
+ hashref => {
+ state => 'WA',
+ district => undef,
+ },
+ addl_from => 'LEFT JOIN cust_main USING (custnum)',
+ extra_sql => sprintf 'AND ( %s ) ', FS::cust_main->active_sql,
+ );
+
+ for my $cust_location ( qsearch( \%qsearch_cust_location )) {
+ local $@;
+ eval {
+ FS::geocode_Mixin::process_district_update(
+ 'FS::cust_location',
+ $cust_location->locationnum
+ );
+ };
+
+ if ( $@ ) {
+ log_error_and_warn(
+ sprintf "Failed to classify district for cust_location(%s): %s",
+ $cust_location->locationnum,
+ $@
+ );
+ } else {
+ log_info_and_warn(
+ sprintf "Classified district for cust_location(%s)",
+ $cust_location->locationnum
+ );
}
- open my $fh, '<', $file
- or die "couldn't open $file: $!\n";
- my $csv = Text::CSV->new;
- my $header = $csv->getline($fh);
- $csv->column_names(@$header);
- # columns we care about are headed 'Code' and 'Rate'
-
- my $total_changed = 0;
- my $total_skipped = 0;
- while ( !$csv->eof ) {
- my $line = $csv->getline_hr($fh);
- my $district = $line->{Code} or next;
- $district = sprintf('%04d', $district);
- my $tax = sprintf('%.1f', $line->{Rate} * 100);
- my $changed = 0;
- my $skipped = 0;
- # find rate(s) in this country+state+district+taxclass that have the
- # wa_sales flag and the configured taxname, and haven't been disabled.
- my @rates = qsearch('cust_main_county', {
- country => 'US',
- state => 'WA', # this is specific to WA
- district => $district,
- taxname => $taxname,
- source => 'wa_sales',
- tax => { op => '>', value => '0' },
- });
- foreach my $rate (@rates) {
- if ( $rate->tax == $tax ) {
- $skipped++;
- } else {
- $rate->set('tax', $tax);
- my $error = $rate->replace;
- die "error updating district $district: $error\n" if $error;
- $changed++;
+
+ sleep 1; # Be polite to WA DOR API
+ }
+
+ for my $cust_location ( qsearch( \%qsearch_cust_location )) {
+ log_error_and_warn(
+ sprintf
+ "Customer address in WA lacking tax district classification. ".
+ "custnum(%s) ".
+ "locationnum(%s) ".
+ "address(%s, %s %s, %s) ".
+ "[https://webgis.dor.wa.gov/taxratelookup/SalesTax.aspx]",
+ map { $cust_location->$_ }
+ qw( custnum locationnum address1 city state zip )
+ );
+ }
+
+}
+
+
+=head2 wa_sales_update_tax_table \%args
+
+Update city/district sales tax rates in L<FS::cust_main_county> from the
+Washington State Department of Revenue published data files.
+
+Creates, or updates, a L<FS::cust_main_county> row for every tax district
+in Washington state. Some cities have different tax rates based on the
+address, within the city. Because of this, some cities have multiple
+districts.
+
+If tax classes are enabled, a row is created in every tax class for
+every district.
+
+Customer addresses aren't classified into districts here. Instead,
+when a Washington state address is inserted or changed in L<FS::cust_location>,
+a job is queued for FS::geocode_Mixin::process_district_update, to ask the
+Washington state API which tax district to use for this address.
+
+All arguments are optional:
+
+ filename: Skip file download, and process the specified filename instead
+
+ taxname: Updated or created records will be set to the given tax name.
+ If not specified, conf value 'tax_district_taxname' is used
+
+ year: Specify year for tax table download. Defaults to current year
+
+ quarter: Specify quarter for tax table download. Defaults to current quarter
+
+=head3 Washington State Department of Revenue Resources
+
+The state of Washington makes data files available via their public website.
+It's possible the availability or format of these files may change. As of now,
+the only data file that contains both city and county names is published in
+XLSX format.
+
+=over 4
+
+=item WA Dept of Revenue
+
+https://dor.wa.gov
+
+=item Data file downloads
+
+https://dor.wa.gov/find-taxes-rates/sales-and-use-tax-rates/downloadable-database
+
+=item XLSX file example
+
+https://dor.wa.gov/sites/default/files/legacy/Docs/forms/ExcsTx/LocSalUseTx/ExcelLocalSlsUserates_19_Q1.xlsx
+
+=item CSV file example
+
+https://dor.wa.gov/sites/default/files/legacy/downloads/Add_DataRates2018Q4.zip
+
+
+=item Address lookup API tool
+
+http://webgis.dor.wa.gov/webapi/AddressRates.aspx?output=xml&addr=410 Terry Ave. North&city=&zip=98100
+
+=back
+
+=cut
+
+sub wa_sales_update_tax_table {
+ my $args = shift;
+
+ croak 'wa_sales_update_tax_table requires \$args hashref'
+ if $args && !ref $args;
+
+ return
+ unless conf_tax_district_method()
+ && conf_tax_district_method() eq 'wa_sales';
+
+ $args->{taxname} ||= FS::Conf->new->config('tax_district_taxname');
+ $args->{year} ||= DateTime->now->year;
+ $args->{quarter} ||= DateTime->now->quarter;
+
+ log_info_and_warn(
+ "Begin wa_sales_update_tax_table() ".
+ join ', ' => (
+ map{ "$_ => ". ( $args->{$_} || 'undef' ) }
+ sort keys %$args
+ )
+ );
+
+ $args->{temp_dir} ||= tempdir();
+
+ $args->{filename} ||= wa_sales_fetch_xlsx_file( $args );
+
+ $args->{tax_districts} = wa_sales_parse_xlsx_file( $args );
+
+ wa_sales_update_cust_main_county( $args );
+
+ log_info_and_warn( 'Finished wa_sales_update_tax_table()' );
+}
+
+=head2 wa_sales_update_cust_main_county \%args
+
+Create or update the L<FS::cust_main_county> records with new data
+
+
+
+=cut
+
+sub wa_sales_update_cust_main_county {
+ my $args = shift;
+
+ return
+ unless conf_tax_district_method()
+ && conf_tax_district_method() eq 'wa_sales';
+
+ croak 'wa_sales_update_cust_main_county requires $args hashref'
+ unless ref $args
+ && ref $args->{tax_districts};
+
+ my $insert_count = 0;
+ my $update_count = 0;
+ my $same_count = 0;
+
+ # Work within a SQL transaction
+ local $FS::UID::AutoCommit = 0;
+
+ for my $taxclass ( FS::part_pkg_taxclass->taxclass_names ) {
+ $taxclass ||= undef; # trap empty string when taxclasses are disabled
+
+ my %cust_main_county =
+ map { $_->district => $_ }
+ qsearch(
+ cust_main_county => {
+ district => { op => '!=', value => undef },
+ state => 'WA',
+ country => 'US',
+ source => 'wa_sales',
+ taxclass => $taxclass,
}
- } # foreach $taxclass
- print "$district: updated $changed, skipped $skipped\n"
- if $DEBUG and ($changed or $skipped);
- $total_changed += $changed;
- $total_skipped += $skipped;
+ );
+
+ for my $district ( @{ $args->{tax_districts} } ) {
+ if ( my $row = $cust_main_county{ $district->{district} } ) {
+
+ # District already exists in this taxclass, update if necessary
+ #
+ # If admin updates value of conf tax_district_taxname, instead of
+ # creating an entire separate set of tax rows with
+ # the new taxname, update the taxname on existing records
+
+ if (
+ $row->tax == ( $district->{tax_combined} * 100 )
+ && $row->taxname eq $args->{taxname}
+ ) {
+ $same_count++;
+ next;
+ }
+
+ $row->city( uc $district->{city} );
+ $row->county( uc $district->{county} );
+ $row->taxclass( $taxclass );
+ $row->taxname( $args->{taxname} || undef );
+ $row->tax( $district->{tax_combined} * 100 );
+
+ if ( my $error = $row->replace ) {
+ dbh->rollback;
+ local $FS::UID::AutoCommit = 1;
+ log_error_and_die(
+ sprintf
+ "Error updating cust_main_county row %s for district %s: %s",
+ $row->taxnum,
+ $district->{district},
+ $error
+ );
+ }
+
+ $update_count++;
+
+ } else {
+
+ # District doesn't exist, create row
+
+ my $row = FS::cust_main_county->new({
+ district => $district->{district},
+ city => uc $district->{city},
+ county => uc $district->{county},
+ state => 'WA',
+ country => 'US',
+ taxclass => $taxclass,
+ taxname => $args->{taxname} || undef,
+ tax => $district->{tax_combined} * 100,
+ source => 'wa_sales',
+ });
+
+ if ( my $error = $row->insert ) {
+ dbh->rollback;
+ local $FS::UID::AutoCommit = 1;
+ log_error_and_die(
+ sprintf
+ "Error inserting cust_main_county row for district %s: %s",
+ $district->{district},
+ $error
+ );
+ }
+
+ $cust_main_county{ $district->{district} } = $row;
+ $insert_count++;
+ }
+
+ } # /foreach $district
+ } # /foreach $taxclass
+
+ dbh->commit;
+
+ local $FS::UID::AutoCommit = 1;
+ log_info_and_warn(
+ sprintf
+ "WA tax table update completed. ".
+ "Inserted %s rows, updated %s rows, identical %s rows",
+ $insert_count,
+ $update_count,
+ $same_count
+ );
+
+}
+
+=head2 wa_sales_parse_xlsx_file \%args
+
+Parse given XLSX file for tax district information
+Return an arrayref of district information hashrefs
+
+=cut
+
+sub wa_sales_parse_xlsx_file {
+ my $args = shift;
+
+ croak 'wa_sales_parse_xlsx_file requires $args hashref containing a filename'
+ unless ref $args
+ && $args->{filename};
+
+ # About the file format:
+ #
+ # The current spreadsheet contains the following @columns.
+ # Rows 1 and 2 are a marquee header
+ # Row 3 is the column labels. We will test these to detect
+ # changes in the data format
+ # Rows 4+ are the tax district data
+ #
+ # The "city" column is being parsed from "Location"
+
+ my @columns = qw( city county district tax_local tax_state tax_combined );
+
+ log_error_and_die( "Unable to access XLSX file: $args->{filename}" )
+ unless -r $args->{filename};
+
+ my $xls_parser = Spreadsheet::XLSX->new( $args->{filename} )
+ or log_error_and_die( "Error parsing XLSX file: $!" );
+
+ my $sheet = $xls_parser->{Worksheet}->[0]
+ or log_error_and_die(" Unable to access worksheet 1 in XLSX file" );
+
+ my $cells = $sheet->{Cells}
+ or log_error_and_die( "Unable to read cells in XLSX file" );
+
+ # Read the column labels and verify
+ my %labels =
+ map{ $columns[$_] => $cells->[2][$_]->{Val} }
+ 0 .. scalar(@columns)-1;
+
+ my %expected_labels = (
+ city => 'Location',
+ county => 'County',
+ district => 'Location Code',
+ tax_local => 'Local Rate',
+ tax_state => 'State Rate',
+ tax_combined => 'Combined Sales Tax',
+ );
+
+ if (
+ my @error_labels =
+ grep { lc $labels{$_} ne lc $expected_labels{$_} }
+ @columns
+ ) {
+ my $error = "Error parsing XLS file - ".
+ "Data format may have been updated with WA DOR! ";
+ $error .= "Expected column $expected_labels{$_}, found $labels{$_}! "
+ for @error_labels;
+ log_error_and_die( $error );
+ }
+
+ # Parse the rows into an array of hashes
+ my @districts;
+ for my $row ( 3..$sheet->{MaxRow} ) {
+ my %district = (
+ map { $columns[$_] => $cells->[$row][$_]->{Val} }
+ 0 .. scalar(@columns)-1
+ );
+
+ if (
+ $district{city}
+ && $district{county}
+ && $district{district} =~ /^\d+$/
+ && $district{tax_local} =~ /^\d?\.\d+$/
+ && $district{tax_state} =~ /^\d?\.\d+$/
+ && $district{tax_combined} =~ /^\d?\.\d+$/
+ ) {
+
+ # For some reason, city may contain line breaks!
+ $district{city} =~ s/[\r\n]//g;
+
+ push @districts, \%district;
+ } else {
+ log_warn_and_warn(
+ "Non-usable row found in spreadsheet:\n" . Dumper( \%district )
+ );
}
- print "Updated $total_changed tax rates.\nSkipped $total_skipped unchanged rates.\n" if $DEBUG;
- dbh->commit;
- } # else $method isn't wa_sales, no other methods exist yet
- '';
+
+ }
+
+ log_error_and_die( "No \@districts found in data file!" )
+ unless @districts;
+
+ log_info_and_warn(
+ sprintf "Parsed %s districts from data file", scalar @districts
+ );
+
+ \@districts;
+
+}
+
+=head2 wa_sales_fetch_xlsx_file \%args
+
+Download data file from WA state DOR to temporary storage,
+return filename
+
+=cut
+
+sub wa_sales_fetch_xlsx_file {
+ my $args = shift;
+
+ return
+ unless conf_tax_district_method()
+ && conf_tax_district_method() eq 'wa_sales';
+
+ croak 'wa_sales_fetch_xlsx_file requires \$args hashref'
+ unless ref $args
+ && $args->{temp_dir};
+
+ my $url_base = 'https://dor.wa.gov'.
+ '/sites/default/files/legacy/Docs/forms/ExcsTx/LocSalUseTx';
+
+ my $year = $args->{year} || DateTime->now->year;
+ my $quarter = $args->{quarter} || DateTime->now->quarter;
+ $year = substr( $year, 2, 2 ) if $year >= 1000;
+
+ my $fn = sprintf( 'ExcelLocalSlsUserates_%s_Q%s.xlsx', $year, $quarter );
+ my $url = "$url_base/$fn";
+
+ my $write_fn = "$args->{temp_dir}/$fn";
+
+ log_info_and_warn( "Begin download from url: $url" );
+
+ my $ua = LWP::UserAgent->new;
+ my $res = $ua->get( $url );
+
+ log_error_and_die( "Download error: ".$res->status_line )
+ unless $res->is_success;
+
+ local $@;
+ eval { write_file( $write_fn, $res->decoded_content ); };
+ log_error_and_die( "Problem writing download to disk: $@" )
+ if $@;
+
+ log_info_and_warn( "Temporary file: $write_fn" );
+ $write_fn;
+
+}
+
+sub log {
+ state $log = FS::Log->new('tax_rate_update');
+ $log;
}
+sub log_info_and_warn {
+ my $log_message = shift;
+ warn "$log_message\n";
+ &log()->info( $log_message );
+}
+
+sub log_warn_and_warn {
+ my $log_message = shift;
+ warn "$log_message\n";
+ &log()->warn( $log_message );
+}
+
+sub log_error_and_die {
+ my $log_message = shift;
+ &log()->error( $log_message );
+ die( "$log_message\n" );
+}
+
+sub log_error_and_warn {
+ my $log_message = shift;
+ warn "$log_message\n";
+ &log()->error( $log_message );
+}
+
+sub conf_tax_district_method {
+ state $tax_district_method = FS::Conf->new->config('tax_district_method');
+ $tax_district_method;
+}
+
+
1;
diff --git a/bin/wa_tax_rate_update b/bin/wa_tax_rate_update
index c50a77771..fef126d34 100644
--- a/bin/wa_tax_rate_update
+++ b/bin/wa_tax_rate_update
@@ -1,4 +1,4 @@
-#!/usr/bin/perl
+#!/usr/bin/env perl
=head1 NAME
@@ -10,27 +10,32 @@ Tool to update city/district sales tax rates in I<cust_main_county> from
the Washington State Department of Revenue website.
Creates, or updates, a L<FS::cust_main_county> row for every tax district
-in Washington state. Some cities have different tax rates based on the
-address, within the city. Because of this, some cities may have multiple
-rows defined.
+in Washington state. Some cities have different tax rates based on the
+address, within the city. Because of this, some cities have
+district.
-When a Washington state address is inserted or changed in L<FS::cust_location>,
+If tax classes are enabled, a row is created in every tax class for
+every district.
+
+Customer addresses aren't classified into districts here. Instead,
+when a Washington state address is inserted or changed in L<FS::cust_location>,
a job is queued for FS::geocode_Mixin::process_district_update, to ask the
Washington state API which tax district to use for this address.
Options:
--f <filename>: Skip downloading, and process the given excel file
+ -f <filename>: Skip downloading, and process the given excel file
--c <taxclass>: Updated or create records within given tax class,
- If not specified, taxclass will be set as NULL
+ -t <taxname>: Updated or created records will be set to the given tax name.
+ If not specified, conf value 'tax_district_taxname' will be used
--t <taxname>: Updated or created records will be set to the given tax name.
- If not specified, conf value 'tax_district_taxname' will be used
+ -y <year>: Specify year for tax table - defaults to current year
--y <year>: Specify year for tax table - defaults to current year
+ -q <quarter>: Specify quarter for tax table - defaults to current quarter
--q <quarter>: Specify quarter for tax table - defaults to current quarter
+ -l <lookup>: Attempt to look up the tax district classification for
+ unclassified cust_location records in Washington. Will
+ notify of records that cannot be classified
=head1 Washington State Department of Revenue Resources
@@ -67,297 +72,76 @@ use warnings;
our $VERSION = '0.02'; # Make Getopt:Std happy
-use Data::Dumper;
-use DateTime;
-use File::Temp 'tempdir';
-use File::Slurp qw(read_file write_file);
use Getopt::Std;
-use LWP::UserAgent;
-use Spreadsheet::XLSX;
-use FS::cust_main_county;
+use FS::Cron::tax_rate_update qw(
+ wa_sales_update_tax_table
+ wa_sales_log_customer_without_tax_district
+);
use FS::Log;
-use FS::Record qw(qsearch qsearchs dbh);
use FS::UID qw(adminsuidsetup);
my %opts;
-getopts( 'c:t:y:q:f:', \%opts );
+getopts( 't:y:q:f:l', \%opts );
+
my $user = shift
or die HELP_MESSAGE();
adminsuidsetup( $user )
or die "bad username '$user'\n";
-my $temp_dir = tempdir();
my $log = FS::Log->new('wa_tax_rate_update');
-$opts{t} ||= FS::Conf->new->config('tax_district_taxname');
-
-log_info_and_warn( "Set taxclass=$opts{c}" ) if $opts{c};
-log_info_and_warn( "Set taxname=$opts{t}" ) if $opts{t};
-
-my $xls_fn = $opts{f} || fetch_xlsx_file();
-my $tax_districts = parse_xlsx_file( $xls_fn );
-
-update_cust_main_county( $tax_districts );
-
-# print Dumper( \%opts );
-# for ( 0..5 ) {
-# print Dumper( $tax_districts->[$_] );
-# }
-
-log_info_and_warn( "Finished" );
-exit;
-
-
-sub update_cust_main_county {
- my $tax_districts = shift;
-
- #
- # Working from an assumption tax tables may be loaded multiple times,
- # each with a different tax class,
- # re: comments on FS::geocode_Mixin::process_district_update
- #
-
- my %cust_main_county =
- map { $_->district => $_ }
- qsearch(
- cust_main_county => {
- district => { op => '!=', value => undef },
- state => 'WA',
- source => 'wa_sales',
- taxclass => $opts{c},
- }
- );
-
- # Work within a SQL transaction
- local $FS::UID::AutoCommit = 0;
-
- my $insert_count = 0;
- my $update_count = 0;
-
- for my $district ( @$tax_districts ) {
- if ( my $row = $cust_main_county{ $district->{district} } ) {
-
- $row->city( uc $district->{city} );
- $row->county( uc $district->{county} );
- $row->taxclass( $opts{c} || undef );
- $row->taxname( $opts{t} || undef );
- $row->tax( $district->{tax_combined} * 100 );
-
- if ( my $error = $row->replace ) {
- dbh->rollback;
- local $FS::UID::AutoCommit = 1;
- log_error_and_die(
- sprintf
- "Error updating cust_main_county row %s for district %s: %s",
- $row->taxnum,
- $district->{district},
- $error
- );
- }
-
- $update_count++;
-
- } else {
- my $row = FS::cust_main_county->new({
- district => $district->{district},
- city => uc $district->{city},
- county => uc $district->{county},
- state => 'WA',
- country => 'US',
- taxclass => $opts{c} || undef,
- taxname => $opts{t} || undef,
- tax => $district->{tax_combined} * 100,
- source => 'wa_sales',
- });
-
- if ( my $error = $row->insert ) {
- dbh->rollback;
- local $FS::UID::AutoCommit = 1;
- log_error_and_die(
- sprintf
- "Error inserting cust_main_county row for district %s: %s",
- $district->{district},
- $error
- );
- }
-
- $cust_main_county{ $district->{district} } = $row;
- $insert_count++;
- }
- }
-
- dbh->commit;
-
- local $FS::UID::AutoCommit = 1;
- log_info_and_warn(
- sprintf
- "WA tax table update completed. Inserted %s rows, updated %s rows",
- $insert_count,
- $update_count
- );
-
-}
-
-sub parse_xlsx_file {
- my $parse_fn = shift;
-
- # About the file format:
- #
- # The current spreadsheet contains the following @columns.
- # Rows 1 and 2 are a marquee header
- # Row 3 is the column labels. We will test these to detect
- # changes in the data format
- # Rows 4+ are the tax district data
- #
- # The "city" column is being parsed from "Location"
-
- my @columns = qw( city county district tax_local tax_state tax_combined );
-
- log_error_and_die( "Unable to access XLSX file: $parse_fn" )
- unless -r $parse_fn;
-
- my $xls_parser = Spreadsheet::XLSX->new( $parse_fn )
- or log_error_and_die( "Error parsing XLSX file: $!" );
-
- my $sheet = $xls_parser->{Worksheet}->[0]
- or log_error_and_die(" Unable to access worksheet 1 in XLSX file" );
-
- my $cells = $sheet->{Cells}
- or log_error_and_die( "Unable to read cells in XLSX file" );
-
- # Read the column labels and verify
- my %labels =
- map{ $columns[$_] => $cells->[2][$_]->{Val} }
- 0 .. scalar(@columns)-1;
-
- my %expected_labels = (
- city => 'Location',
- county => 'County',
- district => 'Location Code',
- tax_local => 'Local Rate',
- tax_state => 'State Rate',
- tax_combined => 'Combined Sales Tax',
- );
-
- if (
- my @error_labels =
- grep { lc $labels{$_} ne lc $expected_labels{$_} }
- @columns
- ) {
- my $error = "Error parsing XLS file - ".
- "Data format may have been updated with WA DOR! ";
- $error .= "Expected column $expected_labels{$_}, found $labels{$_}! "
- for @error_labels;
- log_error_and_die( $error );
- }
-
- # Parse the rows into an array of hashes
- my @districts;
- for my $row ( 3..$sheet->{MaxRow} ) {
- my %district = (
- map { $columns[$_] => $cells->[$row][$_]->{Val} }
- 0 .. scalar(@columns)-1
- );
-
- if (
- $district{city}
- && $district{county}
- && $district{district} =~ /^\d+$/
- && $district{tax_local} =~ /^\d?\.\d+$/
- && $district{tax_state} =~ /^\d?\.\d+$/
- && $district{tax_combined} =~ /^\d?\.\d+$/
- ) {
-
- # For some reason, city may contain line breaks!
- $district{city} =~ s/[\r\n]//g;
-
- push @districts, \%district;
- } else {
- log_warn_and_warn(
- "Non-usable row found in spreadsheet:\n" . Dumper( \%district )
- );
- }
+$log->info('Begin wa_tax_rate_update');
+{
+ local $@;
+ eval {
+ wa_sales_update_tax_table({
+ $opts{f} ? ( filename => $opts{f} ) : (),
+ $opts{t} ? ( taxname => $opts{t} ) : (),
+ $opts{y} ? ( year => $opts{y} ) : (),
+ $opts{q} ? ( quarter => $opts{q} ) : (),
+ });
+ };
+
+ if ( $@ ) {
+ $log->error( "Error: $@" );
+ warn "Error: $@\n";
+ } else {
+ $log->info( 'Finished wa_tax_rate_update' );
+ warn "Finished wa_tax_rate_update\n";
}
-
- log_error_and_die( "No \@districts found in data file!" )
- unless @districts;
-
- log_info_and_warn(
- sprintf "Parsed %s districts from data file", scalar @districts
- );
-
- \@districts;
}
-sub fetch_xlsx_file {
- # Download file to temporary storage, return filename
-
- my $url_base = 'https://dor.wa.gov'.
- '/sites/default/files/legacy/Docs/forms/ExcsTx/LocSalUseTx';
-
- my $year = $opts{y} || DateTime->now->year;
- my $quarter = $opts{q} || DateTime->now->quarter;
- $year = substr( $year, 2, 2 ) if $year >= 1000;
- my $fn = sprintf( 'ExcelLocalSlsUserates_%s_Q%s.xlsx', $year, $quarter );
- my $url = "$url_base/$fn";
-
- my $write_fn = "$temp_dir/$fn";
-
- log_info_and_warn( "Begin download from url: $url" );
- my $ua = LWP::UserAgent->new;
- my $res = $ua->get( $url );
+if ( $opts{l} ) {
+ $log->info( 'Begin wa_sales_log_customer_without_tax_district' );
- log_error_and_die( "Download error: ".$res->status_line )
- unless $res->is_success;
+ wa_sales_log_customer_without_tax_district();
- local $@;
- eval { write_file( $write_fn, $res->decoded_content ); };
- log_error_and_die( "Problem writing download to disk: $@" )
- if $@;
-
- log_info_and_warn( "Temporary file: $write_fn" );
- $write_fn;
+ $log->info( 'Finished wa_sales_log_customer_without_tax_district' );
+ warn "Finished wa_sales_log_customer_without_tax_district\n";
}
+exit;
sub HELP_MESSAGE {
print "
Tool to update city/district sales tax rates in I<cust_main_county> from
the Washington State Department of Revenue website.
- Usage: [-f filename] [-c taxclass] [-t taxname] [-y year] [-q quarter]
+ Usage: [-f filename] [-t taxname] [-y year] [-q quarter] [-l] freeside_username
Optional Options:
-f filename Skip download, and process the specified filename
- -c taxclass Update or create records within this taxclass
- defaults as NULL
-t taxname Apply tax name value to created or updated records
defaults as conf value 'tax_district_taxname'
-y year Year for data file download
-q quarter Quarter of data file to download
+ -t lookup Try to fix cust_location records without a district
";
exit;
}
-sub log_info_and_warn {
- my $log_message = shift;
- warn "$log_message\n";
- $log->info( $log_message );
-}
-
-sub log_warn_and_warn {
- my $log_message = shift;
- warn "$log_message\n";
- $log->warn( $log_message );
-}
-
-sub log_error_and_die {
- my $log_message = shift;
- $log->error( $log_message );
- die( "$log_message\n" );
-}
commit 68433955458415f8a1c48629e736e6d4bcf7a985
Author: Mitch Jackson <mitch at freeside.biz>
Date: Tue Dec 18 17:47:57 2018 -0500
RT# 80488 Allow city select for tax_district_method=wa_sales
diff --git a/httemplate/elements/city.html b/httemplate/elements/city.html
index 05250fef5..3c5e91782 100644
--- a/httemplate/elements/city.html
+++ b/httemplate/elements/city.html
@@ -153,9 +153,7 @@ my %opt = @_;
my $pre = $opt{'prefix'};
my $conf = new FS::Conf;
-# Using tax_district_method implies that there's not a preloaded city/county
-# tax district table.
-my $disable_select = 1 if $conf->config('tax_district_method');
+my $disable_select = 0;
$opt{'disable_empty'} = 1 unless exists($opt{'disable_empty'});
commit 584e97b0dc65ac774900fc848e308e540c8dd1a6
Author: Mitch Jackson <mitch at freeside.biz>
Date: Tue Dec 18 17:45:18 2018 -0500
RT# 80488 Find tax districts for WA addresses using official API
diff --git a/FS/FS/Misc/Geo.pm b/FS/FS/Misc/Geo.pm
index 6b3d6ca71..96ce0764b 100644
--- a/FS/FS/Misc/Geo.pm
+++ b/FS/FS/Misc/Geo.pm
@@ -14,6 +14,7 @@ use Data::Dumper;
use FS::Conf;
use FS::Log;
use Locale::Country;
+use XML::LibXML;
FS::UID->install_callback( sub {
$conf = new FS::Conf;
@@ -141,102 +142,155 @@ sub get_district {
&$method($location);
}
-sub wa_sales {
- my $location = shift;
- my $error = '';
- return '' if $location->{state} ne 'WA';
- my $return = { %$location };
- $return->{'exempt_amount'} = 0.00;
+=head2 wa_sales location_hash
- my $url = 'http://webgis2.dor.wa.gov/TaxRateLookup_AGS/TaxReport.aspx';
- my $ua = new LWP::UserAgent;
+Expects output of location_hash() as parameter
+
+Dies on error, or if tax rate cannot be found using given address
+
+Query the WA State Dept of Revenue API with an address, and return
+tax district information for that address.
+
+Documentation for the API can be found here:
+
+L<https://dor.wa.gov/find-taxes-rates/retail-sales-tax/destination-based-sales-tax-and-streamlined-sales-tax/wa-sales-tax-rate-lookup-url-interface>
+
+This API does not return consistent usable county names, as the county
+name may include appreviations or labels referring to PTBA (public transport
+benefit area) or CEZ (community empowerment zone). It's recommended to use
+the tool wa_tax_rate_update to fully populate the city/county/districts for
+WA state every financial quarter.
+
+Returns a hashref with the following keys:
- my $delim = '<|>'; # yes, <|>
- my $year = (localtime)[5] + 1900;
- my $month = (localtime)[4] + 1;
- my @zip = split('-', $location->{zip});
-
- my @args = (
- 'TaxType=S', #sales; 'P' = property
- 'Src=0', #does something complicated
- 'TAXABLE=',
- 'Addr='.uri_escape($location->{address1}),
- 'City='.uri_escape($location->{city}),
- 'Zip='.$zip[0],
- 'Zip1='.($zip[1] || ''), #optional
- 'Year='.$year,
- 'SYear='.$year,
- 'Month='.$month,
- 'EMon='.$month,
+ - district the wa state tax district id
+ - tax the combined total tax rate, as a percentage
+ - city the API rate name
+ - county The API address PTBA
+ - state WA
+ - country US
+ - exempt_amount 0
+
+=cut
+
+sub wa_sales {
+ my $location_hash = shift;
+
+ # Return without die() when called with pointless context
+ return
+ unless $location_hash
+ && ref $location_hash
+ && $location_hash->{state} eq 'WA'
+ && $location_hash->{address1}
+ && $location_hash->{zip}
+ && $location_hash->{city};
+
+ my $log = FS::Log->new('wa_sales');
+
+ warn "wa_sales() called with location_hash:\n".Dumper( $location_hash)."\n"
+ if $DEBUG;
+
+ my $api_url = 'http://webgis.dor.wa.gov/webapi/AddressRates.aspx';
+ my @api_response_codes = (
+ 'The address was found',
+ 'The address was not found, but the ZIP+4 was located.',
+ 'The address was updated and found, the user should validate the address record',
+ 'The address was updated and Zip+4 located, the user should validate the address record',
+ 'The address was corrected and found, the user should validate the address record',
+ 'Neither the address or ZIP+4 was found, but the 5-digit ZIP was located.',
+ 'The address, ZIP+4, and ZIP could not be found.',
+ 'Invalid Latitude/Longitude',
+ 'Internal error'
);
-
- my $query_string = join($delim, @args );
- $url .= "?$query_string";
- warn "\nrequest: $url\n\n" if $DEBUG > 1;
- my $res = $ua->request( GET( "$url?$query_string" ) );
+ my %get_query = (
+ output => 'xml',
+ addr => $location_hash->{address1},
+ city => $location_hash->{city},
+ zip => substr( $location_hash->{zip}, 0, 5 ),
+ );
+ my $get_string = join '&' => (
+ map{ sprintf "%s=%s", $_, uri_escape( $get_query{$_} ) }
+ keys %get_query
+ );
- warn $res->as_string
- if $DEBUG > 2;
+ my $prepared_url = "${api_url}?$get_string";
- if ($res->code ne '200') {
- $error = $res->message;
- }
+ warn "API call to URL: $prepared_url\n"
+ if $DEBUG;
- my $content = $res->content;
- my $p = new HTML::TokeParser \$content;
- my $js = '';
- while ( my $t = $p->get_tag('script') ) {
- my $u = $p->get_token; #either enclosed text or the </script> tag
- if ( $u->[0] eq 'T' and $u->[1] =~ /tblSales/ ) {
- $js = $u->[1];
- last;
- }
+ my $dom;
+ local $@;
+ eval { $dom = XML::LibXML->load_xml( location => $prepared_url ); };
+ if ( $@ ) {
+ my $error =
+ sprintf "Problem parsing XML from API URL(%s): %s",
+ $prepared_url, $@;
+ $log->error( $error );
+ die $error;
}
- if ( $js ) { #found it
- # strip down to the quoted string, which contains escaped single quotes.
- $js =~ s/.*\('tblSales'\);c.innerHTML='//s;
- $js =~ s/(?<!\\)'.*//s; # (?<!\\) means "not preceded by a backslash"
- warn "\n\n innerHTML:\n$js\n\n" if $DEBUG > 2;
-
- $p = new HTML::TokeParser \$js;
- TD: while ( my $td = $p->get_tag('td') ) {
- while ( my $u = $p->get_token ) {
- next TD if $u->[0] eq 'E' and $u->[1] eq 'td';
- next if $u->[0] ne 'T'; # skip non-text
- my $text = $u->[1];
-
- if ( lc($text) eq 'location code' ) {
- $p->get_tag('td'); # skip to the next column
- undef $u;
- $u = $p->get_token until ($u->[0] || '') eq 'T'; # and then skip non-text
- $return->{'district'} = $u->[1];
- }
- elsif ( lc($text) eq 'total tax rate' ) {
- $p->get_tag('td');
- undef $u;
- $u = $p->get_token until ($u->[0] || '') eq 'T';
- $return->{'tax'} = $u->[1];
- }
- } # get_token
- } # TD
-
- # just to make sure
- if ( $return->{'district'} =~ /^\d+$/ and $return->{'tax'} =~ /^.\d+$/ ) {
- $return->{'tax'} *= 100; #percentage
- warn Dumper($return) if $DEBUG > 1;
- return $return;
- }
- else {
- $error = 'district code/tax rate not found';
- }
+
+ my ($res_root) = $dom->findnodes('/response');
+ my ($res_addressline) = $dom->findnodes('/response/addressline');
+ my ($res_rate) = $dom->findnodes('/response/rate');
+
+ my $res_code = $res_root->getAttribute('code')
+ if $res_root;
+
+ unless (
+ ref $res_root
+ && ref $res_addressline
+ && ref $res_rate
+ && $res_code <= 5
+ && $res_root->getAttribute('rate') > 0
+ ) {
+ my $error =
+ sprintf
+ "Problem querying WA DOR tax district - " .
+ "code( %s %s ) " .
+ "address( %s ) " .
+ "url( %s )",
+ $res_code || 'n/a',
+ $res_code ? $api_response_codes[$res_code] : 'n/a',
+ $location_hash->{address1},
+ $prepared_url;
+ $log->error( $error );
+ die "$error\n";
}
- else {
- $error = "failed to parse document";
+
+ my %response = (
+ exempt_amount => 0,
+ state => 'WA',
+ country => 'US',
+ district => $res_root->getAttribute('loccode'),
+ tax => $res_root->getAttribute('rate') * 100,
+ county => uc $res_addressline->getAttribute('ptba'),
+ city => uc $res_rate->getAttribute('name')
+ );
+
+ $response{county} =~ s/ PTBA//i;
+
+ if ( $DEBUG ) {
+ warn "XML document: $dom\n";
+ warn "API parsed response: ".Dumper( \%response )."\n";
}
- die "WA tax district lookup error: $error";
+ my $info_message =
+ sprintf
+ "Tax district(%s) selected for address(%s %s %s %s)",
+ $response{district},
+ $location_hash->{address1},
+ $location_hash->{city},
+ $location_hash->{state},
+ $location_hash->{zip};
+
+ $log->info( $info_message );
+ warn "$info_message\n"
+ if $DEBUG;
+
+ \%response;
+
}
###### USPS Standardization ######
commit 09ab7084366594c1fd76eae1375aca75db774ba8
Author: Mitch Jackson <mitch at freeside.biz>
Date: Tue Dec 18 03:20:20 2018 -0500
RT# 80488 Utility to populate WA tax district table
diff --git a/bin/wa_tax_rate_update b/bin/wa_tax_rate_update
index d4a4b52e4..c50a77771 100644
--- a/bin/wa_tax_rate_update
+++ b/bin/wa_tax_rate_update
@@ -9,118 +9,355 @@ wa_tax_rate_update
Tool to update city/district sales tax rates in I<cust_main_county> from
the Washington State Department of Revenue website.
-This does not handle address standardization or geocoding addresses to
-Washington tax district codes. That logic is still in FS::Misc::Geo,
-and relies on a heinous screen-scraping of the interactive search tool.
-This script just updates the cust_main_county records that already exist
-with the latest quarterly tax rates.
+Creates, or updates, a L<FS::cust_main_county> row for every tax district
+in Washington state. Some cities have different tax rates based on the
+address, within the city. Because of this, some cities may have multiple
+rows defined.
+
+When a Washington state address is inserted or changed in L<FS::cust_location>,
+a job is queued for FS::geocode_Mixin::process_district_update, to ask the
+Washington state API which tax district to use for this address.
Options:
--c <taxclass>: operate only on records with the named tax class. If not
-specified, this operates on records with null tax class.
+-f <filename>: Skip downloading, and process the given excel file
+
+-c <taxclass>: Updated or create records within given tax class,
+ If not specified, taxclass will be set as NULL
+
+-t <taxname>: Updated or created records will be set to the given tax name.
+ If not specified, conf value 'tax_district_taxname' will be used
+
+-y <year>: Specify year for tax table - defaults to current year
+
+-q <quarter>: Specify quarter for tax table - defaults to current quarter
+
+=head1 Washington State Department of Revenue Resources
+
+The state of Washington makes data files available via their public website.
+It's possible the availability or format of these files may change. As of now,
+the only data file that contains both city and county names is published in
+XLSX format.
+
+=item WA Dept of Revenue
+
+https://dor.wa.gov
+
+=item Data file downloads
+
+https://dor.wa.gov/find-taxes-rates/sales-and-use-tax-rates/downloadable-database
+
+=item XLSX file example
--t <taxname>: operate only on records with that tax name. If not specified,
-it operates on records where the tax name is either null or 'Tax'.
+https://dor.wa.gov/sites/default/files/legacy/Docs/forms/ExcsTx/LocSalUseTx/ExcelLocalSlsUserates_19_Q1.xlsx
+
+=item CSV file example
+
+https://dor.wa.gov/sites/default/files/legacy/downloads/Add_DataRates2018Q4.zip
+
+
+=item Address lookup API tool
+
+http://webgis.dor.wa.gov/webapi/AddressRates.aspx?output=xml&addr=410 Terry Ave. North&city=&zip=98100
=cut
-use FS::Record qw(qsearch qsearchs dbh);
-use FS::cust_main_county;
-use FS::UID qw(adminsuidsetup);
+use strict;
+use warnings;
+
+our $VERSION = '0.02'; # Make Getopt:Std happy
+
+use Data::Dumper;
use DateTime;
-use LWP::UserAgent;
use File::Temp 'tempdir';
use File::Slurp qw(read_file write_file);
-use Text::CSV;
use Getopt::Std;
+use LWP::UserAgent;
+use Spreadsheet::XLSX;
-getopts('c:t:');
-my $user = shift or die usage();
-
-# download the update file
-my $now = DateTime->now;
-my $yr = $now->year;
-my $qt = $now->quarter;
-my $file = "Rates${yr}Q${qt}.zip";
-my $url = 'http://dor.wa.gov/downloads/Add_Data/'.$file;
-my $dir = tempdir();
-chdir($dir);
-my $ua = LWP::UserAgent->new;
-warn "Downloading $url...\n";
-my $response = $ua->get($url);
-if ( ! $response->is_success ) {
- die $response->status_line;
-}
-write_file($file, $response->decoded_content);
+use FS::cust_main_county;
+use FS::Log;
+use FS::Record qw(qsearch qsearchs dbh);
+use FS::UID qw(adminsuidsetup);
+
+my %opts;
+getopts( 'c:t:y:q:f:', \%opts );
+my $user = shift
+ or die HELP_MESSAGE();
+
+adminsuidsetup( $user )
+ or die "bad username '$user'\n";
+
+my $temp_dir = tempdir();
+my $log = FS::Log->new('wa_tax_rate_update');
+
+$opts{t} ||= FS::Conf->new->config('tax_district_taxname');
+
+log_info_and_warn( "Set taxclass=$opts{c}" ) if $opts{c};
+log_info_and_warn( "Set taxname=$opts{t}" ) if $opts{t};
+
+my $xls_fn = $opts{f} || fetch_xlsx_file();
+my $tax_districts = parse_xlsx_file( $xls_fn );
+
+update_cust_main_county( $tax_districts );
+
+# print Dumper( \%opts );
+# for ( 0..5 ) {
+# print Dumper( $tax_districts->[$_] );
+# }
+
+log_info_and_warn( "Finished" );
+exit;
+
+
+sub update_cust_main_county {
+ my $tax_districts = shift;
+
+ #
+ # Working from an assumption tax tables may be loaded multiple times,
+ # each with a different tax class,
+ # re: comments on FS::geocode_Mixin::process_district_update
+ #
+
+ my %cust_main_county =
+ map { $_->district => $_ }
+ qsearch(
+ cust_main_county => {
+ district => { op => '!=', value => undef },
+ state => 'WA',
+ source => 'wa_sales',
+ taxclass => $opts{c},
+ }
+ );
+
+ # Work within a SQL transaction
+ local $FS::UID::AutoCommit = 0;
+
+ my $insert_count = 0;
+ my $update_count = 0;
+
+ for my $district ( @$tax_districts ) {
+ if ( my $row = $cust_main_county{ $district->{district} } ) {
+
+ $row->city( uc $district->{city} );
+ $row->county( uc $district->{county} );
+ $row->taxclass( $opts{c} || undef );
+ $row->taxname( $opts{t} || undef );
+ $row->tax( $district->{tax_combined} * 100 );
+
+ if ( my $error = $row->replace ) {
+ dbh->rollback;
+ local $FS::UID::AutoCommit = 1;
+ log_error_and_die(
+ sprintf
+ "Error updating cust_main_county row %s for district %s: %s",
+ $row->taxnum,
+ $district->{district},
+ $error
+ );
+ }
+
+ $update_count++;
+
+ } else {
+ my $row = FS::cust_main_county->new({
+ district => $district->{district},
+ city => uc $district->{city},
+ county => uc $district->{county},
+ state => 'WA',
+ country => 'US',
+ taxclass => $opts{c} || undef,
+ taxname => $opts{t} || undef,
+ tax => $district->{tax_combined} * 100,
+ source => 'wa_sales',
+ });
+
+ if ( my $error = $row->insert ) {
+ dbh->rollback;
+ local $FS::UID::AutoCommit = 1;
+ log_error_and_die(
+ sprintf
+ "Error inserting cust_main_county row for district %s: %s",
+ $district->{district},
+ $error
+ );
+ }
+
+ $cust_main_county{ $district->{district} } = $row;
+ $insert_count++;
+ }
+ }
+
+ dbh->commit;
+
+ local $FS::UID::AutoCommit = 1;
+ log_info_and_warn(
+ sprintf
+ "WA tax table update completed. Inserted %s rows, updated %s rows",
+ $insert_count,
+ $update_count
+ );
-# parse it
-system('unzip', $file);
-$file =~ s/\.zip$/.csv/;
-if (! -f $file) {
- die "$file not found in zip archive.\n";
}
-open my $fh, '<', $file
- or die "couldn't open $file: $!\n";
-my $csv = Text::CSV->new;
-my $header = $csv->getline($fh);
-$csv->column_names(@$header);
-# columns we care about are headed 'Code' and 'Rate'
-
-# connect to the DB
-adminsuidsetup($user) or die "bad username '$user'\n";
-$FS::UID::AutoCommit = 0;
-
-$opt_c ||= ''; # taxclass
-$opt_t ||= ''; # taxname
-my $total_changed = 0;
-my $total_skipped = 0;
-while ( !$csv->eof ) {
- my $line = $csv->getline_hr($fh);
- my $district = $line->{Code} or next;
- $district = sprintf('%04d', $district);
- my $tax = sprintf('%.1f', $line->{Rate} * 100);
- my $changed = 0;
- my $skipped = 0;
- # find all rates in WA
- my @rates = qsearch('cust_main_county', {
- country => 'US',
- state => 'WA', # this is specific to WA
- district => $district,
- taxclass => $opt_c,
- taxname => $opt_t,
- tax => { op => '>', value => '0' },
- });
- if ($opt_t eq '') {
- push @rates, qsearch('cust_main_county', {
- country => 'US',
- state => 'WA', # this is specific to WA
- district => $district,
- taxclass => $opt_c,
- taxname => 'Tax',
- tax => { op => '>', value => '0' },
- });
+
+sub parse_xlsx_file {
+ my $parse_fn = shift;
+
+ # About the file format:
+ #
+ # The current spreadsheet contains the following @columns.
+ # Rows 1 and 2 are a marquee header
+ # Row 3 is the column labels. We will test these to detect
+ # changes in the data format
+ # Rows 4+ are the tax district data
+ #
+ # The "city" column is being parsed from "Location"
+
+ my @columns = qw( city county district tax_local tax_state tax_combined );
+
+ log_error_and_die( "Unable to access XLSX file: $parse_fn" )
+ unless -r $parse_fn;
+
+ my $xls_parser = Spreadsheet::XLSX->new( $parse_fn )
+ or log_error_and_die( "Error parsing XLSX file: $!" );
+
+ my $sheet = $xls_parser->{Worksheet}->[0]
+ or log_error_and_die(" Unable to access worksheet 1 in XLSX file" );
+
+ my $cells = $sheet->{Cells}
+ or log_error_and_die( "Unable to read cells in XLSX file" );
+
+ # Read the column labels and verify
+ my %labels =
+ map{ $columns[$_] => $cells->[2][$_]->{Val} }
+ 0 .. scalar(@columns)-1;
+
+ my %expected_labels = (
+ city => 'Location',
+ county => 'County',
+ district => 'Location Code',
+ tax_local => 'Local Rate',
+ tax_state => 'State Rate',
+ tax_combined => 'Combined Sales Tax',
+ );
+
+ if (
+ my @error_labels =
+ grep { lc $labels{$_} ne lc $expected_labels{$_} }
+ @columns
+ ) {
+ my $error = "Error parsing XLS file - ".
+ "Data format may have been updated with WA DOR! ";
+ $error .= "Expected column $expected_labels{$_}, found $labels{$_}! "
+ for @error_labels;
+ log_error_and_die( $error );
}
- foreach my $rate (@rates) {
- if ( $rate->tax == $tax ) {
- $skipped++;
+
+ # Parse the rows into an array of hashes
+ my @districts;
+ for my $row ( 3..$sheet->{MaxRow} ) {
+ my %district = (
+ map { $columns[$_] => $cells->[$row][$_]->{Val} }
+ 0 .. scalar(@columns)-1
+ );
+
+ if (
+ $district{city}
+ && $district{county}
+ && $district{district} =~ /^\d+$/
+ && $district{tax_local} =~ /^\d?\.\d+$/
+ && $district{tax_state} =~ /^\d?\.\d+$/
+ && $district{tax_combined} =~ /^\d?\.\d+$/
+ ) {
+
+ # For some reason, city may contain line breaks!
+ $district{city} =~ s/[\r\n]//g;
+
+ push @districts, \%district;
} else {
- $rate->set('tax', $tax);
- my $error = $rate->replace;
- die "error updating district $district: $error\n" if $error;
- $changed++;
+ log_warn_and_warn(
+ "Non-usable row found in spreadsheet:\n" . Dumper( \%district )
+ );
}
+
}
- print "$district: updated $changed, skipped $skipped\n"
- if $changed or $skipped;
- $total_changed += $changed;
- $total_skipped += $skipped;
+
+ log_error_and_die( "No \@districts found in data file!" )
+ unless @districts;
+
+ log_info_and_warn(
+ sprintf "Parsed %s districts from data file", scalar @districts
+ );
+
+ \@districts;
+}
+
+sub fetch_xlsx_file {
+ # Download file to temporary storage, return filename
+
+ my $url_base = 'https://dor.wa.gov'.
+ '/sites/default/files/legacy/Docs/forms/ExcsTx/LocSalUseTx';
+
+ my $year = $opts{y} || DateTime->now->year;
+ my $quarter = $opts{q} || DateTime->now->quarter;
+ $year = substr( $year, 2, 2 ) if $year >= 1000;
+ my $fn = sprintf( 'ExcelLocalSlsUserates_%s_Q%s.xlsx', $year, $quarter );
+ my $url = "$url_base/$fn";
+
+ my $write_fn = "$temp_dir/$fn";
+
+ log_info_and_warn( "Begin download from url: $url" );
+
+ my $ua = LWP::UserAgent->new;
+ my $res = $ua->get( $url );
+
+ log_error_and_die( "Download error: ".$res->status_line )
+ unless $res->is_success;
+
+ local $@;
+ eval { write_file( $write_fn, $res->decoded_content ); };
+ log_error_and_die( "Problem writing download to disk: $@" )
+ if $@;
+
+ log_info_and_warn( "Temporary file: $write_fn" );
+ $write_fn;
+}
+
+
+sub HELP_MESSAGE {
+ print "
+ Tool to update city/district sales tax rates in I<cust_main_county> from
+ the Washington State Department of Revenue website.
+
+ Usage: [-f filename] [-c taxclass] [-t taxname] [-y year] [-q quarter]
+
+ Optional Options:
+ -f filename Skip download, and process the specified filename
+ -c taxclass Update or create records within this taxclass
+ defaults as NULL
+ -t taxname Apply tax name value to created or updated records
+ defaults as conf value 'tax_district_taxname'
+ -y year Year for data file download
+ -q quarter Quarter of data file to download
+
+ ";
+ exit;
+}
+
+sub log_info_and_warn {
+ my $log_message = shift;
+ warn "$log_message\n";
+ $log->info( $log_message );
+}
+
+sub log_warn_and_warn {
+ my $log_message = shift;
+ warn "$log_message\n";
+ $log->warn( $log_message );
}
-print "Updated $total_changed tax rates.\nSkipped $total_skipped unchanged rates.\n";
-dbh->commit;
-sub usage {
- "usage:
- wa_tax_rate_update [ -c taxclass ] [ -t taxname ] user
-";
+sub log_error_and_die {
+ my $log_message = shift;
+ $log->error( $log_message );
+ die( "$log_message\n" );
}
-----------------------------------------------------------------------
Summary of changes:
FS/FS/Cron/tax_rate_update.pm | 673 ++++++++++++++++++++++++++++++++++++------
FS/FS/Misc/Geo.pm | 235 +++++++++------
bin/wa_tax_rate_update | 223 +++++++-------
debian/control | 3 +-
httemplate/elements/city.html | 4 +-
5 files changed, 867 insertions(+), 271 deletions(-)
mode change 100644 => 100755 bin/wa_tax_rate_update
More information about the freeside-commits
mailing list