[freeside-commits] freeside/bin import-did-inventory,1.1,1.2
Erik Levinson
levinse at wavetail.420.am
Fri May 20 23:32:41 PDT 2011
Update of /home/cvs/cvsroot/freeside/bin
In directory wavetail.420.am:/tmp/cvs-serv9612/bin
Modified Files:
import-did-inventory
Log Message:
did inventory importing, RT12754
Index: import-did-inventory
===================================================================
RCS file: /home/cvs/cvsroot/freeside/bin/import-did-inventory,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -w -d -r1.1 -r1.2
--- import-did-inventory 21 May 2011 00:52:48 -0000 1.1
+++ import-did-inventory 21 May 2011 06:32:39 -0000 1.2
@@ -16,8 +16,11 @@
use FS::rate_center;
use FS::phone_avail;
use FS::did_vendor;
+use FS::svc_phone;
use Data::Dumper;
+print "started time=".time."\n";
+
#### SET THESE! #################################
my $file = '/home/levinse/dids1.csv';
my $did_vendor_id = 1;
@@ -46,6 +49,7 @@
my %did_order = ();
my %rate_center = ();
my %rate_center_abbrev = ();
+my %cust2pkg = ();
my %msamap = (
# YOU CANNOT USE THE STATE/NPA/LATA OF A DID TO TRY TO FIND ITS MSA. IT HAS
# NOTHING IN COMMON WITH THE STATE OF THE MSA. THERE IS SIMPLY INSUFFICIENT
@@ -68,13 +72,16 @@
'Portland ME' => 38860,
);
my $skipto = 0;
-my $limit = 0;
+my $limit = 900;
my $linenum = 1;
-# cache the whole LATA table in one query for performance
+# cache LATA and MSA tables in one query for performance
my @latas = qsearch('lata', {});
my %latas = map { $_->latanum => $_->description } @latas;
+my @msas = qsearch('msa', {});
+my %msas = map { $_->msanum => $_->description } @msas;
+
# now add in the brain-dead LATA hacks
$latas{636} = 'BRAINERD-FARGO ND';
$latas{920} = 'CONNECTICUT';
@@ -84,18 +91,76 @@
$latas{952} = 'TAMPA FLORIDA';
$latas{524} = 'KANSAS CITY';
-
+my $parser = new DateTime::Format::Natural( 'time_zone' => 'local' );
sub parsedt {
my ($dt,$min,$max) = (shift,shift,shift);
- my $parser = new DateTime::Format::Natural( 'time_zone' => 'local' );
my $epoch = $parser->parse_datetime($dt);
return $epoch->epoch
if ($parser->success && $epoch->epoch >= $min && $epoch->epoch <= $max);
fatal("invalid date $dt (min=$min, max=$max)");
}
-sub latacheck {
- my ($latanum,$latadesc) = (shift,shift);
+sub msatest {
+ my ($their,$our) = (shift,shift);
+ my $a = $our;
+ $a =~ s/,.*?$//;
+ return 1 if $a eq $their;
+ return 1 if ($our =~ /^([\w\s]+)-/ && $1 eq $their);
+ 0;
+}
+
+sub trim {
+ my $str = shift;
+ $str =~ s/^\s+|\s+$//g;
+ $str;
+}
+
+sub suffer {
+ my $linenum = shift;
+ my @columns = @_;
+
+ my $did = trim($columns[0]);
+ my $npa = trim($columns[1]);
+ my $state = trim($columns[2]);
+ my $rate_center_abbrev = trim($columns[3]);
+ my $rate_center = trim($columns[4]);
+ my $customer = trim($columns[5]);
+ my $submitted = parsedt(trim($columns[7]),$min_date,$max_date);
+
+ my $ordernum = trim($columns[8]);
+ return if $ordernum eq 'Unknown';
+
+ my $confirmed = parsedt(trim($columns[9]),$submitted,$max_date);
+
+ # sometimes, we're in a non-Y2K-compliant bullshit format, differing from
+ # all the other dates. Other times, we randomly change formats multiple times
+ # in the middle of the file for absolutely no reason...wtf
+ my $received = trim($columns[10]);
+ if ( $received =~ /^(\d{1,2})\/(\d{1,2})\/(\d{2})$/ ) {
+ $received = $2."/".$1."/20".$3;
+ } elsif ( $received !~ /^\d{2}\/\d{2}\/\d{4}$/ ) {
+ fatal("invalid received date $received");
+ }
+ if ( $ordernum == 300383 ) { # another hack due to bad data
+ $received = parsedt($received,1,$max_date)
+ } else {
+ $received = parsedt($received,$confirmed,$max_date);
+ }
+
+ my $latanum = trim($columns[12]);
+ my $latadesc = trim($columns[13]);
+ my $msadesc = trim($columns[14]);
+
+ fatal("invalid DID and/or NPA or NPA doesn't match DID")
+ unless ($did =~ /^(\d{3})\d{7}$/ && $npa == $1);
+ fatal("invalid state, order #, LATA #, or LATA description")
+ unless ($state =~ /^[A-Z]{2}$/ && ($ordernum =~ /^\d+$/ || $ordernum eq 'Test') # more hacks
+ && $latanum =~ /^\d{3}$/
+ && $latadesc =~ /^[\w\s\-]+$/);
+
+
+ ### LATA ###
+
fatal("no lata found for latanum $latanum") unless exists($latas{$latanum});
# unsurprisingly, our idea of a LATA name doesn't always match their idea
@@ -112,12 +177,10 @@
fatal("their LATA description '$latadesc' doesn't match our LATA description '$ourdesc'")
unless uc($ourdesc) eq uc($latadesc);
-}
-# XXX: performance
-sub msacheck {
- my $msadesc = shift;
- my $state = shift;
+
+ ### MSA ###
+
my $msanum = -1;
# XXX: no idea what the MSA is for Danbury, so discard it for now and deal with it manually/later
@@ -135,111 +198,33 @@
$msanum = $msamap{$msadesc};
}
else {
- my @msa = qsearch('msa', { 'description' => {
- 'op' => 'ILIKE',
- 'value' => "$msadesc%"
- }
- });
-
- # so now we have two cases for a match and everything else is a non-match
- foreach my $msa ( @msa ) {
- # a. our MSA stripped of state portion matches their MSA exactly
- my $msatest1 = $msa->description;
- $msatest1 =~ s/,.*?$//;
- if($msatest1 eq $msadesc) {
- fatal("multiple MSA matches (case 1) for $msadesc") unless $msanum == -1;
- $msanum = $msa->msanum;
- }
-
- # b. our MSA stripped of state portion and up to the first hyphen matches their MSA exactly
- my $msatest2 = $msa->description;
- if($msatest2 =~ /^([\w\s]+)-/ && $1 eq $msadesc) {
- fatal("multiple MSA matches (case 2) for $msadesc") unless $msanum == -1;
- $msanum = $msa->msanum;
- }
- }
-
+ my @msa = grep { msatest($msadesc,$_->description) } @msas;
+ fatal("multiple MSA matches for '$msadesc'") if(scalar(@msa) > 1);
+ $msanum = $msa[0]->msanum if scalar(@msa) == 1;
$msamap{$msadesc} = $msanum if $msanum != -1;
}
fatal("msa $msadesc not found") if $msanum == -1;
warn "$msadesc matched msanum $msanum for line $linenum\n" if $debug;
}
- $msanum;
-}
-sub ratecentercheck {
- my ($rate_center, $rate_center_abbrev) = (shift,shift);
+ ### RATE CENTER ###
if ( exists $rate_center{$rate_center} ) {
fatal("rate center abbreviation for '$rate_center' doesn't exist or doesn't match '$rate_center_abbrev'")
unless ( exists $rate_center_abbrev{$rate_center} &&
$rate_center_abbrev{$rate_center} eq $rate_center_abbrev);
} else {
+ print "creating new rate center '$rate_center' '$rate_center_abbrev'\n";
my $rc = new FS::rate_center{ description => $rate_center };
my $error = $rc->insert;
- fatal("can't insert rate center '$rate_center': $error") if $error;
+ fatal("can't insert rate center '$rate_center' '$rate_center_abbrev': $error")
+ if $error;
$rate_center{$rate_center} = $rc->ratecenternum;
$rate_center_abbrev{$rate_center} = $rate_center_abbrev;
}
+ my $ratecenternum = $rate_center{$rate_center};
- $rate_center{$rate_center};
-}
-
-sub trim {
- my $str = shift;
- $str =~ s/^\s+|\s+$//g;
- $str;
-}
-
-sub suffer {
- my $linenum = shift;
- my @columns = @_;
-
- my $did = trim($columns[0]);
- my $npa = trim($columns[1]);
- my $state = trim($columns[2]);
- my $rate_center_abbrev = trim($columns[3]);
- my $rate_center = trim($columns[4]);
- my $customer = trim($columns[5]);
- my $submitted = parsedt(trim($columns[7]),$min_date,$max_date);
-
- my $ordernum = trim($columns[8]);
- return if $ordernum eq 'Unknown';
-
- my $confirmed = parsedt(trim($columns[9]),$submitted,$max_date);
-
- # sometimes, we're in a non-Y2K-compliant bullshit format, differing from
- # all the other dates. Other times, we randomly change formats multiple times
- # in the middle of the file for absolutely no reason...wtf
- my $received = trim($columns[10]);
- if ( $received =~ /^(\d{1,2})\/(\d{1,2})\/(\d{2})$/ ) {
- $received = $2."/".$1."/20".$3;
- } elsif ( $received !~ /^\d{2}\/\d{2}\/\d{4}$/ ) {
- fatal("invalid received date $received");
- }
- if ( $ordernum == 300383 ) { # another hack due to bad data
- $received = parsedt($received,1,$max_date)
- } else {
- $received = parsedt($received,$confirmed,$max_date);
- }
-
- my $latanum = trim($columns[12]);
- my $latadesc = trim($columns[13]);
- my $msadesc = trim($columns[14]);
-
- fatal("invalid DID and/or NPA or NPA doesn't match DID")
- unless ($did =~ /^(\d{3})\d{7}$/ && $npa == $1);
- fatal("invalid state, order #, LATA #, or LATA description")
- unless ($state =~ /^[A-Z]{2}$/ && ($ordernum =~ /^\d+$/ || $ordernum eq 'Test') # more hacks
- && $latanum =~ /^\d{3}$/
- && $latadesc =~ /^[\w\s\-]+$/);
-
- latacheck($latanum,$latadesc);
- my $msanum = msacheck($msadesc,$state);
- my $ratecenternum = ratecentercheck($rate_center,$rate_center_abbrev);
-
- # now we passed most basic checks/lookups (and possibly created a rate center)
my $order = order($ordernum,$submitted,$confirmed,$received,$customer);
my $order_item = order_item($order,$npa,$latanum,$state,$msanum,$ratecenternum);
@@ -249,7 +234,7 @@
warn "Pass $linenum\n" if $debug;
my $time = time;
- warn "Done $linenum time=$time\n" if ($linenum % 100 == 0);
+ print "Done $linenum time=$time\n" if ($linenum % 100 == 0);
}
sub phone_avail {
@@ -280,7 +265,6 @@
$pa;
}
-# XXX: performance
sub order_item {
my($order,$npa,$latanum,$state,$msanum,$ratecenternum)
= (shift,shift,shift,shift,shift,shift);
@@ -329,13 +313,40 @@
if ( exists($custname2num{$customer}) ) {
$cust{'custnum'} = $custname2num{$customer};
} else {
+ print "new customer case for '$customer'\n";
my @cust_main = smart_search('search' => $customer);
fatal(scalar(@cust_main) . " customers found for $customer")
unless scalar(@cust_main) == 1;
- $cust{'custnum'} = $cust_main[0]->custnum;
+ my $cust_main = $cust_main[0];
- # cache it, or we'll be going even slower than we already are
- $custname2num{$customer} = $cust_main[0]->custnum;
+ $cust{'custnum'} = $cust_main->custnum;
+ $custname2num{$customer} = $cust_main->custnum;
+ $cust2pkg{$cust_main->custnum} = {};
+
+ my @pkgs = $cust_main->ncancelled_pkgs;
+ fatal("no packages") unless scalar(@pkgs);
+
+ foreach my $pkg ( @pkgs ) {
+ my @avail_part_svc = $pkg->available_part_svc;
+ my @svcpart;
+ foreach my $avail_part_svc ( @avail_part_svc ) {
+ if ($avail_part_svc->svcdb eq 'svc_phone') {
+ push @svcpart, $avail_part_svc->svcpart;
+ }
+ }
+ fatal("multiple svc_phone services") if scalar(@svcpart) > 1;
+ fatal("multiple packages with svc_phone services")
+ if (exists $cust2pkg{$cust_main->custnum}->{pkgnum}
+ && scalar(@svcpart));
+ if(scalar(@svcpart) == 1) {
+ $cust2pkg{$cust_main->custnum}->{pkgnum} = $pkg->pkgnum;
+ $cust2pkg{$cust_main->custnum}->{svcpart} = $svcpart[0];
+ }
+ }
+
+ fatal("no pkg/svc")
+ unless (exists $cust2pkg{$cust_main->custnum}->{pkgnum}
+ && exists $cust2pkg{$cust_main->custnum}->{svcpart});
}
}
@@ -367,9 +378,6 @@
$o;
}
-# XXX: damn it...this sub increases the import time by a factor of THREE!
-# should probably modify and run it in batch for each customer, as opposed to per DID
-# if import finishes in under two hours, leaving as-is
sub provision {
my($did,$customer,$phone_avail) = (shift,shift,shift);
@@ -377,45 +385,27 @@
# because of the above, we now need to do the internal did db
# export's job ourselves (set the svcnum for the DID in phone_avail)
- my $cust_main = qsearchs('cust_main', { custnum => $custname2num{$customer} })
- || fatal('invalid customer');
-
- my @pkgs = $cust_main->ncancelled_pkgs;
- fatal("no packages") unless scalar(@pkgs);
-
- my $thepkg;
- my $svcpart;
- foreach my $pkg ( @pkgs ) {
- my @avail_part_svc = $pkg->available_part_svc;
- my @svcpart;
- foreach my $avail_part_svc ( @avail_part_svc ) {
- if ($avail_part_svc->svcdb eq 'svc_phone') {
- push @svcpart, $avail_part_svc->svcpart;
- }
- }
- fatal("multiple svc_phone services") if scalar(@svcpart) > 1;
- fatal("multiple packages with svc_phone services")
- if ($thepkg && scalar(@svcpart));
- if(scalar(@svcpart) == 1) {
- $thepkg = $pkg;
- $svcpart = $svcpart[0];
- }
- }
-
- fatal("no pkg/svc") unless ($thepkg && $svcpart);
+ fatal("customer not found") unless exists $cust2pkg{$custname2num{$customer}};
my $svc_phone = new FS::svc_phone({
- pkgnum => $thepkg->pkgnum,
- svcpart => $svcpart,
+ pkgnum => $cust2pkg{$custname2num{$customer}}->{pkgnum},
+ svcpart => $cust2pkg{$custname2num{$customer}}->{svcpart},
countrycode => 1,
phonenum => $did,
});
+
+ # XXX: THIS LINE CAUSES PERFORMANCE TO DEGRADE AND THE SCRIPT NEVER FINISHES! WHY!?
+ # -unattaching the exports has no effect
+ # -after each successive call, the time taken to complete 100 rows becomes greater
+ # -commenting out this call results in a constant time taken to complete 100 rows
+ # -after 10K rows we get a random error that makes no sense
my $error = $svc_phone->insert;
+
fatal("can't insert svc_phone: $error") if $error;
- $phone_avail->svcnum($svc_phone->svcnum);
- $error = $phone_avail->replace;
- fatal("can't replace phone_avail: $error") if $error;
+ #$phone_avail->svcnum($svc_phone->svcnum);
+ #$error = $phone_avail->replace;
+ #fatal("can't replace phone_avail: $error") if $error;
'';
}
@@ -428,7 +418,7 @@
my $csv = new Text::CSV;
open (CSV, "<", $file) or die $!;
-warn "Starting main loop time=".time;
+print "Starting main loop time=".time."\n";
while (<CSV>) {
if ( $linenum == 1 ) { # skip header
$linenum++;
@@ -451,8 +441,8 @@
suffer($linenum, at columns);
} else {
my $err = $csv->error_diag . "(" . $csv->error_input . ")";
- warn "WARNING: failed to parse line $linenum: " . $csv->error_diag
- . " (" . $csv->error_input . ")";
+ print "WARNING: failed to parse line $linenum: " . $csv->error_diag
+ . " (" . $csv->error_input . ")\n";
}
$linenum++;
}
More information about the freeside-commits
mailing list