[freeside-commits] freeside/bin import-did-inventory,NONE,1.1
Erik Levinson
levinse at wavetail.420.am
Fri May 20 17:52:50 PDT 2011
Update of /home/cvs/cvsroot/freeside/bin
In directory wavetail.420.am:/tmp/cvs-serv11712/bin
Added Files:
import-did-inventory
Log Message:
DID inventory import, RT12754
--- NEW FILE: import-did-inventory ---
#!/usr/bin/perl
use strict;
use warnings;
use Text::CSV;
use FS::UID qw(adminsuidsetup);
use FS::cust_main_county;
use FS::Record qw(qsearch qsearchs dbh);
use DateTime::Format::Natural;
use FS::lata;
use FS::msa;
use FS::cust_main;
use FS::cust_main::Search qw(smart_search);
use FS::did_order;
use FS::did_order_item;
use FS::rate_center;
use FS::phone_avail;
use FS::did_vendor;
use Data::Dumper;
#### SET THESE! #################################
my $file = '/home/levinse/dids1.csv';
my $did_vendor_id = 1;
my $dry = 1;
my $debug = 0;
my $internal_diddb_exportnum = 2; # IMPORTANT: set this to the correct exportnum or everything will go in wrong into phone_avail
my %custname2num = (); # MyCust => 12345,
################################################
my $user = shift;
adminsuidsetup $user;
# oh yeah this is fun when you can't Ctrl+C me
local $SIG{HUP} = 'IGNORE';
local $SIG{INT} = 'IGNORE';
local $SIG{QUIT} = 'IGNORE';
local $SIG{TERM} = 'IGNORE';
local $SIG{TSTP} = 'IGNORE';
local $SIG{PIPE} = 'IGNORE';
my $oldAutoCommit = $FS::UID::AutoCommit;
local $FS::UID::AutoCommit = 0;
my $dbh = dbh;
my $max_date = time;
my $min_date = 1262304000; # January 1st 2010
my %did_order = ();
my %rate_center = ();
my %rate_center_abbrev = ();
my %msamap = (
# YOU CANNOT USE THE STATE/NPA/LATA OF A DID TO TRY TO FIND ITS MSA. IT HAS
# NOTHING IN COMMON WITH THE STATE OF THE MSA. THERE IS SIMPLY INSUFFICIENT
# DATA IN THE CSV FILE TO DETERMINE CANONICAL MSA WITHOUT THIS:
'Washington DC' => 47900,
'Fort Lauderdale' => 33100,
'Cambridge' => 14460,
'Boise' => 14260,
'New York' => 35620,
'Aberdeen' => 10100,
'Bloomington' => 14020,
'Las Vegas' => 29820,
'Madison' => 31540,
'Miami' => 33100,
'Jackson' => 27140,
'St Cloud' => 41060,
# more hax upon hax (the above are unique, no issues)
'Portland OR' => 38900,
'Portland ME' => 38860,
);
my $skipto = 0;
my $limit = 0;
my $linenum = 1;
# cache the whole LATA table in one query for performance
my @latas = qsearch('lata', {});
my %latas = map { $_->latanum => $_->description } @latas;
# now add in the brain-dead LATA hacks
$latas{636} = 'BRAINERD-FARGO ND';
$latas{920} = 'CONNECTICUT';
$latas{334} = 'AUBURN-HUNTINGTON IN';
$latas{232} = 'NORTHEAST - PA';
$latas{460} = 'SOUTHEAST FL GR-EA';
$latas{952} = 'TAMPA FLORIDA';
$latas{524} = 'KANSAS CITY';
sub parsedt {
my ($dt,$min,$max) = (shift,shift,shift);
my $parser = new DateTime::Format::Natural( 'time_zone' => 'local' );
my $epoch = $parser->parse_datetime($dt);
return $epoch->epoch
if ($parser->success && $epoch->epoch >= $min && $epoch->epoch <= $max);
fatal("invalid date $dt (min=$min, max=$max)");
}
sub latacheck {
my ($latanum,$latadesc) = (shift,shift);
fatal("no lata found for latanum $latanum") unless exists($latas{$latanum});
# unsurprisingly, our idea of a LATA name doesn't always match their idea
# of the same. Specifically, they randomly expand the state portion and
# abbreviate it arbitrarily
my $ourdesc = $latas{$latanum};
# strip off the fixed state abbreviation portion in ours
$ourdesc =~ s/ ..$//;
# strip off the variable state abbreviation (or full name) portion in theirs
$latadesc =~ s/\s\w+$// unless uc($ourdesc) eq uc($latadesc); # yeah...long story :(
fatal("their LATA description '$latadesc' doesn't match our LATA description '$ourdesc'")
unless uc($ourdesc) eq uc($latadesc);
}
# XXX: performance
sub msacheck {
my $msadesc = shift;
my $state = shift;
my $msanum = -1;
# XXX: no idea what the MSA is for Danbury, so discard it for now and deal with it manually/later
$msadesc = '' if $msadesc eq 'Danbury';
# hax on hax
$msadesc = 'Portland OR' if ($msadesc eq 'Portland' && $state eq 'OR');
$msadesc = 'Portland ME' if ($msadesc eq 'Portland' && $state eq 'ME');
# not everything in their file has a MSA
if ( $msadesc =~ /^[\w\s]+$/ ) {
# their idea of a MSA differs from our idea of it
if ( exists($msamap{$msadesc}) ) {
$msanum = $msamap{$msadesc};
}
else {
my @msa = qsearch('msa', { 'description' => {
'op' => 'ILIKE',
'value' => "$msadesc%"
}
});
# so now we have two cases for a match and everything else is a non-match
foreach my $msa ( @msa ) {
# a. our MSA stripped of state portion matches their MSA exactly
my $msatest1 = $msa->description;
$msatest1 =~ s/,.*?$//;
if($msatest1 eq $msadesc) {
fatal("multiple MSA matches (case 1) for $msadesc") unless $msanum == -1;
$msanum = $msa->msanum;
}
# b. our MSA stripped of state portion and up to the first hyphen matches their MSA exactly
my $msatest2 = $msa->description;
if($msatest2 =~ /^([\w\s]+)-/ && $1 eq $msadesc) {
fatal("multiple MSA matches (case 2) for $msadesc") unless $msanum == -1;
$msanum = $msa->msanum;
}
}
$msamap{$msadesc} = $msanum if $msanum != -1;
}
fatal("msa $msadesc not found") if $msanum == -1;
warn "$msadesc matched msanum $msanum for line $linenum\n" if $debug;
}
$msanum;
}
sub ratecentercheck {
my ($rate_center, $rate_center_abbrev) = (shift,shift);
if ( exists $rate_center{$rate_center} ) {
fatal("rate center abbreviation for '$rate_center' doesn't exist or doesn't match '$rate_center_abbrev'")
unless ( exists $rate_center_abbrev{$rate_center} &&
$rate_center_abbrev{$rate_center} eq $rate_center_abbrev);
} else {
my $rc = new FS::rate_center{ description => $rate_center };
my $error = $rc->insert;
fatal("can't insert rate center '$rate_center': $error") if $error;
$rate_center{$rate_center} = $rc->ratecenternum;
$rate_center_abbrev{$rate_center} = $rate_center_abbrev;
}
$rate_center{$rate_center};
}
sub trim {
my $str = shift;
$str =~ s/^\s+|\s+$//g;
$str;
}
sub suffer {
my $linenum = shift;
my @columns = @_;
my $did = trim($columns[0]);
my $npa = trim($columns[1]);
my $state = trim($columns[2]);
my $rate_center_abbrev = trim($columns[3]);
my $rate_center = trim($columns[4]);
my $customer = trim($columns[5]);
my $submitted = parsedt(trim($columns[7]),$min_date,$max_date);
my $ordernum = trim($columns[8]);
return if $ordernum eq 'Unknown';
my $confirmed = parsedt(trim($columns[9]),$submitted,$max_date);
# sometimes, we're in a non-Y2K-compliant bullshit format, differing from
# all the other dates. Other times, we randomly change formats multiple times
# in the middle of the file for absolutely no reason...wtf
my $received = trim($columns[10]);
if ( $received =~ /^(\d{1,2})\/(\d{1,2})\/(\d{2})$/ ) {
$received = $2."/".$1."/20".$3;
} elsif ( $received !~ /^\d{2}\/\d{2}\/\d{4}$/ ) {
fatal("invalid received date $received");
}
if ( $ordernum == 300383 ) { # another hack due to bad data
$received = parsedt($received,1,$max_date)
} else {
$received = parsedt($received,$confirmed,$max_date);
}
my $latanum = trim($columns[12]);
my $latadesc = trim($columns[13]);
my $msadesc = trim($columns[14]);
fatal("invalid DID and/or NPA or NPA doesn't match DID")
unless ($did =~ /^(\d{3})\d{7}$/ && $npa == $1);
fatal("invalid state, order #, LATA #, or LATA description")
unless ($state =~ /^[A-Z]{2}$/ && ($ordernum =~ /^\d+$/ || $ordernum eq 'Test') # more hacks
&& $latanum =~ /^\d{3}$/
&& $latadesc =~ /^[\w\s\-]+$/);
latacheck($latanum,$latadesc);
my $msanum = msacheck($msadesc,$state);
my $ratecenternum = ratecentercheck($rate_center,$rate_center_abbrev);
# now we passed most basic checks/lookups (and possibly created a rate center)
my $order = order($ordernum,$submitted,$confirmed,$received,$customer);
my $order_item = order_item($order,$npa,$latanum,$state,$msanum,$ratecenternum);
my $phone_avail = phone_avail($order,$state,$did,$rate_center,$latanum,$msanum);
provision($did,$customer,$phone_avail) if $customer ne 'Stock';
warn "Pass $linenum\n" if $debug;
my $time = time;
warn "Done $linenum time=$time\n" if ($linenum % 100 == 0);
}
sub phone_avail {
my ($order,$state,$did,$rate_center,$latanum,$msanum)
= (shift,shift,shift,shift,shift,shift);
$did =~ /^(\d{3})(\d{3})(\d{4})$/;
my $npa = $1;
my $nxx = $2;
my $station = $3;
my %hash = (
exportnum => $internal_diddb_exportnum,
countrycode => '1',
state => $state,
npa => $npa,
nxx => $nxx,
station => $station,
name => $rate_center,
rate_center_abbrev => $rate_center_abbrev{$rate_center},
ordernum => $order->ordernum,
latanum => $latanum,
);
$hash{'msanum'} = $msanum if $msanum != -1;
my $pa = new FS::phone_avail{ %hash };
my $error = $pa->insert;
fatal("can't insert phone_avail: $error") if $error;
$pa;
}
# XXX: performance
sub order_item {
my($order,$npa,$latanum,$state,$msanum,$ratecenternum)
= (shift,shift,shift,shift,shift,shift);
my %msa = ();
$msa{'msanum'} = $msanum if $msanum != -1;
my $oi;
my @order_item = $order->did_order_item;
foreach my $order_item ( @order_item ) {
if($order_item->npa == $npa
&& $order_item->latanum == $latanum
&& $order_item->state eq $state
&& $order_item->ratecenternum == $ratecenternum
&& (!$order_item->msanum || $order_item->msanum == $msanum) ) {
fatal("Multiple order items") if $oi;
$oi = $order_item;
}
}
if($oi) {
$oi->quantity($oi->quantity+1);
my $error = $oi->replace;
fatal("can't replace order item: $error") if $error;
} else {
$oi = new FS::did_order_item{ ordernum => $order->ordernum,
npa => $npa,
latanum => $latanum,
state => $state,
quantity => 1,
ratecenternum => $ratecenternum,
%msa, };
my $error = $oi->insert;
fatal("can't insert order item: $error") if $error;
}
fatal("wtf2") unless $oi;
$oi;
}
sub order {
my($vendor_order_id,$submitted,$confirmed,$received,$customer)
= (shift,shift,shift,shift,shift);
my %cust = ();
if ( $customer ne 'Stock' ) {
if ( exists($custname2num{$customer}) ) {
$cust{'custnum'} = $custname2num{$customer};
} else {
my @cust_main = smart_search('search' => $customer);
fatal(scalar(@cust_main) . " customers found for $customer")
unless scalar(@cust_main) == 1;
$cust{'custnum'} = $cust_main[0]->custnum;
# cache it, or we'll be going even slower than we already are
$custname2num{$customer} = $cust_main[0]->custnum;
}
}
my $o;
if( exists $did_order{$vendor_order_id} ) {
$o = $did_order{$vendor_order_id};
fatal("vendor order #$vendor_order_id - order data differs from one item to another")
unless ( ($o->submitted == $submitted
|| $o->vendor_order_id == 293011) # yet another bad data hack
&& $o->confirmed == $confirmed
&& $o->received == $received);
fatal("customer mismatch for vendor order #$vendor_order_id")
unless ( ($o->custnum && $cust{'custnum'}
&& $o->custnum == $cust{'custnum'})
|| (!$o->custnum && !exists($cust{'custnum'})) );
} else {
$o = new FS::did_order{ vendornum => $did_vendor_id,
vendor_order_id => $vendor_order_id,
submitted => $submitted,
confirmed => $confirmed,
received => $received,
%cust, };
my $error = $o->insert;
fatal("can't insert vendor order #$vendor_order_id: $error") if $error;
$did_order{$vendor_order_id} = $o;
}
fatal("wtf") unless $o;
$o;
}
# XXX: damn it...this sub increases the import time by a factor of THREE!
# should probably modify and run it in batch for each customer, as opposed to per DID
# if import finishes in under two hours, leaving as-is
sub provision {
my($did,$customer,$phone_avail) = (shift,shift,shift);
local $FS::svc_Common::noexport_hack = 1;
# because of the above, we now need to do the internal did db
# export's job ourselves (set the svcnum for the DID in phone_avail)
my $cust_main = qsearchs('cust_main', { custnum => $custname2num{$customer} })
|| fatal('invalid customer');
my @pkgs = $cust_main->ncancelled_pkgs;
fatal("no packages") unless scalar(@pkgs);
my $thepkg;
my $svcpart;
foreach my $pkg ( @pkgs ) {
my @avail_part_svc = $pkg->available_part_svc;
my @svcpart;
foreach my $avail_part_svc ( @avail_part_svc ) {
if ($avail_part_svc->svcdb eq 'svc_phone') {
push @svcpart, $avail_part_svc->svcpart;
}
}
fatal("multiple svc_phone services") if scalar(@svcpart) > 1;
fatal("multiple packages with svc_phone services")
if ($thepkg && scalar(@svcpart));
if(scalar(@svcpart) == 1) {
$thepkg = $pkg;
$svcpart = $svcpart[0];
}
}
fatal("no pkg/svc") unless ($thepkg && $svcpart);
my $svc_phone = new FS::svc_phone({
pkgnum => $thepkg->pkgnum,
svcpart => $svcpart,
countrycode => 1,
phonenum => $did,
});
my $error = $svc_phone->insert;
fatal("can't insert svc_phone: $error") if $error;
$phone_avail->svcnum($svc_phone->svcnum);
$error = $phone_avail->replace;
fatal("can't replace phone_avail: $error") if $error;
'';
}
sub fatal {
my $msg = shift;
$dbh->rollback if $oldAutoCommit;
die $msg;
}
my $csv = new Text::CSV;
open (CSV, "<", $file) or die $!;
warn "Starting main loop time=".time;
while (<CSV>) {
if ( $linenum == 1 ) { # skip header
$linenum++;
next;
}
if( $skipto > $linenum ) { # debug stuff
$linenum++;
next;
}
last if $limit > 0 && $limit <= $linenum;
# kept getting these errors for many lines:
# "EIQ - Binary character inside quoted field, binary off"
$_ =~ s/[^[:ascii:]]//g;
if ($csv->parse($_)) {
my @columns = $csv->fields();
suffer($linenum, at columns);
} else {
my $err = $csv->error_diag . "(" . $csv->error_input . ")";
warn "WARNING: failed to parse line $linenum: " . $csv->error_diag
. " (" . $csv->error_input . ")";
}
$linenum++;
}
close CSV;
fatal("COMMIT ABORTED DUE TO DRY RUN BEING ON") if $dry;
$dbh->commit or die $dbh->errstr if $oldAutoCommit;
More information about the freeside-commits
mailing list