[freeside-commits] branch FREESIDE_3_BRANCH updated. 104fd2deee705abe830e537fb892e1d993900836
Jonathan Prykop
jonathan at 420.am
Thu Dec 3 21:22:21 PST 2015
The branch, FREESIDE_3_BRANCH has been updated
via 104fd2deee705abe830e537fb892e1d993900836 (commit)
via e70fc3984596c904457a2e6c6dc4908d15635ddf (commit)
via 95b039903f6c046fd591cf16031639d0f6a29c87 (commit)
via 8026055be161f8e42310b9cac7278b8a537996b8 (commit)
from 9511d92e8337497cf6f0877c56c7b677de7b2ba5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 104fd2deee705abe830e537fb892e1d993900836
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Dec 3 23:15:34 2015 -0600
RT#37901: Galactic Telecom CDRs [overlap is better than a gap]
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
index 9a02a72..5b28e3f 100644
--- a/FS/bin/freeside-cdr-portaone-import
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -53,7 +53,7 @@ unless ($startdate) {
'hashref' => { 'cdrbatch' => {op=>'like', value=>'portaone-import%'}},
'order_by' => 'ORDER BY _date DESC LIMIT 1',
});
- $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date + 1) if $lastbatch;
+ $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date) if $lastbatch;
}
$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
commit e70fc3984596c904457a2e6c6dc4908d15635ddf
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Dec 3 22:13:31 2015 -0600
RT#37901: Galactic Telecom CDRs [skip duplicates, midnight start]
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
index d9a606a..9a02a72 100644
--- a/FS/bin/freeside-cdr-portaone-import
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -7,9 +7,12 @@ use Date::Parse 'str2time';
use Getopt::Long;
use JSON;
use Net::HTTPS::Any qw(https_post);
+use Time::Local;
use FS::Record qw(qsearchs dbh);
use FS::UID qw(adminsuidsetup);
+use FS::cdr;
+use FS::cdr_batch;
sub usage {
"Usage:
@@ -54,7 +57,8 @@ unless ($startdate) {
}
$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
-my $now = time;
+my @now = localtime();
+my $now = timelocal(0,0,0,$now[3],$now[4],$now[5]); #most recent midnight
if ($enddate) {
$enddate = str2time($enddate) or die "Can't parse enddate $enddate";
$now = $enddate;
@@ -65,8 +69,11 @@ $enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
$FS::UID::AutoCommit = 0;
+my $cdrbatchname = 'portaone-import-'. time2str('%Y/%m/%d-%T',$now);
+die "Batch $cdrbatchname already exists, please specify a different end date. " . usage()
+ if FS::cdr_batch->row_exists('cdrbatch = ?', $cdrbatchname);
my $cdr_batch = new FS::cdr_batch({
- 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ 'cdrbatch' => $cdrbatchname,
'_date' => $now,
});
my $error = $cdr_batch->insert;
@@ -102,8 +109,13 @@ foreach my $custnum (@custnum) {
'offset' => $offset,
});
my @xdrs = @{$xdrs->{'xdr_list'}};
- print "Inserting ". at xdrs." records\n" if $verbose && @xdrs;
+ print "Retrieved ". at xdrs." records\n" if $verbose && @xdrs;
+ my $skipped = 0; # for verbose display only
foreach my $xdr (@xdrs) {
+ if ( FS::cdr->row_exists('uniqueid = ?', $xdr->{'i_xdr'}) ) {
+ $skipped += 1;
+ next;
+ }
my $desc = $xdr->{'country'};
if ($xdr->{'subdivision'}) {
$desc = ', ' . $desc if $desc;
@@ -131,11 +143,12 @@ foreach my $custnum (@custnum) {
die "Error inserting cdr: $error";
}
} #foreach $xdr
- $totalcount += @xdrs;
+ print "Skipped $skipped duplicate records\n" if $verbose && $skipped;
+ $totalcount += @xdrs - $skipped;
$lastcount = @xdrs;
$offset += $step;
} #while $lastcount == $step
- print scalar($totalcount) . " xdrs retrieved\n" if $verbose;
+ print scalar($totalcount) . " records inserted\n" if $verbose;
} #foreach $custnum
call_api('Session','logout',$auth_info);
commit 95b039903f6c046fd591cf16031639d0f6a29c87
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Thu Dec 3 16:45:15 2015 -0600
RT#37901: Galactic Telecom CDRs [field mappings]
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
index be2555b..d9a606a 100644
--- a/FS/bin/freeside-cdr-portaone-import
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -62,6 +62,19 @@ if ($enddate) {
}
$enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
+
+$FS::UID::AutoCommit = 0;
+
+my $cdr_batch = new FS::cdr_batch({
+ 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ '_date' => $now,
+});
+my $error = $cdr_batch->insert;
+if ($error) {
+ dbh->rollback;
+ die "Error creating batch: $error";
+}
+
print "Downloading records from $startdate to $enddate\n" if $verbose;
my $auth_info; # needs to be declared undef for call_api
@@ -74,58 +87,61 @@ my $results = {};
my $custlist = call_api('Customer','get_customer_list');
my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}};
foreach my $custnum (@custnum) {
- print "Processing customer $custnum\n" if $verbose;
- my $xdrs = call_api('Customer','get_customer_xdrs',{
- 'i_customer' => $custnum,
- 'from_date' => $startdate,
- 'to_date' => $enddate,
- 'cdr_entity' => 'C', # can also be 'A', or blank for both
- });
- my @xdrs = @{$xdrs->{'xdr_list'}};
- print scalar(@xdrs) . " xdrs retrieved\n" if $verbose;
- $results->{$custnum} = \@xdrs;
-}
+ print "Retrieving for customer $custnum\n" if $verbose;
+ my $step = 500; # too many records was crashing server, so we request in chunks
+ my $lastcount = $step; # to get the while loop rolling
+ my $totalcount = 0; # for verbose display only
+ my $offset = 0;
+ while ($lastcount == $step) {
+ my $xdrs = call_api('Customer','get_customer_xdrs',{
+ 'i_customer' => $custnum,
+ 'from_date' => $startdate,
+ 'to_date' => $enddate,
+ 'cdr_entity' => 'A',
+ 'limit' => $step,
+ 'offset' => $offset,
+ });
+ my @xdrs = @{$xdrs->{'xdr_list'}};
+ print "Inserting ". at xdrs." records\n" if $verbose && @xdrs;
+ foreach my $xdr (@xdrs) {
+ my $desc = $xdr->{'country'};
+ if ($xdr->{'subdivision'}) {
+ $desc = ', ' . $desc if $desc;
+ $desc = $xdr->{'subdivision'} . $desc;
+ }
+ if ($xdr->{'description'}) {
+ $desc = ' (' . $desc . ')' if $desc;
+ $desc = $xdr->{'description'} . $desc;
+ }
+ my $cdr = FS::cdr->new ({
+ 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
+ 'uniqueid' => $xdr->{'i_xdr'},
+ 'src' => $xdr->{'CLI'},
+ 'dst' => $xdr->{'CLD'},
+ 'upstream_price' => $xdr->{'charged_amount'},
+ 'startdate' => $xdr->{'unix_connect_time'},
+ 'enddate' => $xdr->{'unix_disconnect_time'},
+ 'accountcode' => $xdr->{'account_id'},
+ 'billsec' => $xdr->{'charged_quantity'},
+ 'upstream_dst_regionname' => $desc,
+ });
+ $error = $cdr->insert;
+ if ($error) {
+ dbh->rollback;
+ die "Error inserting cdr: $error";
+ }
+ } #foreach $xdr
+ $totalcount += @xdrs;
+ $lastcount = @xdrs;
+ $offset += $step;
+ } #while $lastcount == $step
+ print scalar($totalcount) . " xdrs retrieved\n" if $verbose;
+} #foreach $custnum
call_api('Session','logout',$auth_info);
-$FS::UID::AutoCommit = 0;
-
-my $cdr_batch = new FS::cdr_batch({
- 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
- '_date' => $now,
-});
-my $error = $cdr_batch->insert;
-if ($error) {
- dbh->rollback;
- die "Error inserting batch: $error";
-}
-
-foreach my $custnum (keys %$results) {
- foreach my $xdr (@{$results->{$custnum}}) {
- my $cdr = FS::cdr->new ({
- 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
- 'acctid' => $xdr->{'i_xdr'}, #???
- 'accountcode' => $xdr->{'account_id'}, #???
- 'startdate' => $xdr->{'unix_connect_time'},
- 'enddate' => $xdr->{'unix_disconnect_time'},
-
-### XXX NEED TO MAP FIELDS:
+### Full list of fields returned by API:
#i_xdr int The unique ID of the xdr record
-#account_id int The unique ID of the account database record
-#CLI string Calling Line Identification
-#CLD string Called Line Identification
-#charged_amount float Amount charged
-#charged_quantity int Units charged
-#country string Country
-#subdivision string Country subdivision
-#description string Destination description
-#disconnect_cause string The code of disconnect cause
-#bill_status string Call bill status
-#disconnect_reason string Call disconnect reason
-#connect_time dateTime Call connect time
-#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
-#disconnect_time dateTime Call disconnect time
-#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record
#account_id int The unique ID of the account database record
#CLI string Calling Line Identification
#CLD string Called Line Identification
@@ -144,21 +160,8 @@ foreach my $custnum (keys %$results) {
#bill_time dateTime Call bill time
#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
#call_recording_url string Path to recorded .wav files
-#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch)
-#bill_time dateTime Call bill time
-#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
-#call_recording_url string Path to recorded .wav files
#call_recording_server_url string URL to the recording server
- });
- $error = $cdr->insert;
- if ($error) {
- dbh->rollback;
- die "Error inserting cdr: $error";
- }
- } #foreach $xdr
-} #foreach $custnum
-
dbh->commit;
exit;
@@ -167,7 +170,7 @@ sub call_api {
my ($service,$method,$params) = @_;
my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : ();
$params ||= {};
- print "Calling $service/$method...\n" if $verbose;
+ print "Calling $service/$method\n" if $verbose;
my ( $page, $response, %reply_headers ) = https_post(
'host' => $host,
'port' => $port,
@@ -175,6 +178,7 @@ sub call_api {
'args' => [ %auth_info, 'params' => encode_json($params) ],
);
return decode_json($page) if $response eq '200 OK';
+ dbh->rollback;
if ($response =~ /^500/) {
my $error = decode_json($page);
die "Server returned error during $service/$method: ".$error->{'faultstring'}
commit 8026055be161f8e42310b9cac7278b8a537996b8
Author: Jonathan Prykop <jonathan at freeside.biz>
Date: Tue Dec 1 02:27:19 2015 -0600
RT37901: Galactic Telecom CDRs [need to finish field mapping
diff --git a/FS/bin/freeside-cdr-portaone-import b/FS/bin/freeside-cdr-portaone-import
new file mode 100644
index 0000000..be2555b
--- /dev/null
+++ b/FS/bin/freeside-cdr-portaone-import
@@ -0,0 +1,186 @@
+#!/usr/bin/perl
+
+use strict;
+
+use Date::Format 'time2str';
+use Date::Parse 'str2time';
+use Getopt::Long;
+use JSON;
+use Net::HTTPS::Any qw(https_post);
+
+use FS::Record qw(qsearchs dbh);
+use FS::UID qw(adminsuidsetup);
+
+sub usage {
+ "Usage:
+freeside-cdr-portaone-import -h 'your.domain.com:443' -u switchusername -p switchpass
+ [-s startdate] [-e enddate] [-v] freesideuser
+";
+}
+
+my ($host,$username,$password,$startdate,$enddate,$verbose);
+GetOptions(
+ "enddate=s" => \$enddate,
+ "host=s" => \$host,
+ "password=s" => \$password,
+ "startdate=s" => \$startdate,
+ "username=s" => \$username,
+ "verbose" => \$verbose,
+);
+
+my $fsuser = $ARGV[-1];
+
+die usage() unless $host && $password && $username && $fsuser;
+
+adminsuidsetup($fsuser);
+
+my $port = 443;
+if ($host =~ /^(.*)\:(.*)$/) {
+ $host = $1;
+ $port = $2;
+}
+
+if ($startdate) {
+ $startdate = str2time($startdate) or die "Can't parse startdate $startdate";
+ $startdate = time2str("%Y-%m-%d %H:%M:%S",$startdate);
+}
+unless ($startdate) {
+ my $lastbatch = qsearchs({
+ 'table' => 'cdr_batch',
+ 'hashref' => { 'cdrbatch' => {op=>'like', value=>'portaone-import%'}},
+ 'order_by' => 'ORDER BY _date DESC LIMIT 1',
+ });
+ $startdate = time2str("%Y-%m-%d %H:%M:%S", $lastbatch->_date + 1) if $lastbatch;
+}
+$startdate ||= '2010-01-01 00:00:00'; #seems decently in the past
+
+my $now = time;
+if ($enddate) {
+ $enddate = str2time($enddate) or die "Can't parse enddate $enddate";
+ $now = $enddate;
+ $enddate = time2str("%Y-%m-%d %H:%M:%S",$enddate);
+}
+$enddate ||= time2str("%Y-%m-%d %H:%M:%S",$now);
+
+print "Downloading records from $startdate to $enddate\n" if $verbose;
+
+my $auth_info; # needs to be declared undef for call_api
+$auth_info = call_api('Session','login',{
+ 'login' => $username,
+ 'password' => $password,
+});
+
+my $results = {};
+my $custlist = call_api('Customer','get_customer_list');
+my @custnum = map { $_->{'i_customer'} } @{$custlist->{'customer_list'}};
+foreach my $custnum (@custnum) {
+ print "Processing customer $custnum\n" if $verbose;
+ my $xdrs = call_api('Customer','get_customer_xdrs',{
+ 'i_customer' => $custnum,
+ 'from_date' => $startdate,
+ 'to_date' => $enddate,
+ 'cdr_entity' => 'C', # can also be 'A', or blank for both
+ });
+ my @xdrs = @{$xdrs->{'xdr_list'}};
+ print scalar(@xdrs) . " xdrs retrieved\n" if $verbose;
+ $results->{$custnum} = \@xdrs;
+}
+
+call_api('Session','logout',$auth_info);
+
+$FS::UID::AutoCommit = 0;
+
+my $cdr_batch = new FS::cdr_batch({
+ 'cdrbatch' => 'portaone-import-'. time2str('%Y/%m/%d-%T',$now),
+ '_date' => $now,
+});
+my $error = $cdr_batch->insert;
+if ($error) {
+ dbh->rollback;
+ die "Error inserting batch: $error";
+}
+
+foreach my $custnum (keys %$results) {
+ foreach my $xdr (@{$results->{$custnum}}) {
+ my $cdr = FS::cdr->new ({
+ 'cdrbatchnum' => $cdr_batch->cdrbatchnum,
+ 'acctid' => $xdr->{'i_xdr'}, #???
+ 'accountcode' => $xdr->{'account_id'}, #???
+ 'startdate' => $xdr->{'unix_connect_time'},
+ 'enddate' => $xdr->{'unix_disconnect_time'},
+
+### XXX NEED TO MAP FIELDS:
+#i_xdr int The unique ID of the xdr record
+#account_id int The unique ID of the account database record
+#CLI string Calling Line Identification
+#CLD string Called Line Identification
+#charged_amount float Amount charged
+#charged_quantity int Units charged
+#country string Country
+#subdivision string Country subdivision
+#description string Destination description
+#disconnect_cause string The code of disconnect cause
+#bill_status string Call bill status
+#disconnect_reason string Call disconnect reason
+#connect_time dateTime Call connect time
+#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
+#disconnect_time dateTime Call disconnect time
+#unix_disconnect_time int Call disconnect time i_xdr int The unique ID of the xdr record
+#account_id int The unique ID of the account database record
+#CLI string Calling Line Identification
+#CLD string Called Line Identification
+#charged_amount float Amount charged
+#charged_quantity int Units charged
+#country string Country
+#subdivision string Country subdivision
+#description string Destination description
+#disconnect_cause string The code of disconnect cause
+#bill_status string Call bill status
+#disconnect_reason string Call disconnect reason
+#connect_time dateTime Call connect time
+#unix_connect_time int Call connect time (expressed in Unix time format - seconds since epoch)
+#disconnect_time dateTime Call disconnect time
+#unix_disconnect_time int Call disconnect time (expressed in Unix time format - seconds since epoch)
+#bill_time dateTime Call bill time
+#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
+#call_recording_url string Path to recorded .wav files
+#call_recording_server_url string URL to the recording server (expressed in Unix time format - seconds since epoch)
+#bill_time dateTime Call bill time
+#bit_flags int Extended information how the service was used; the integer field that should be treated as a bit-map. Each currently used bit is listed in the Transaction_Flag_Types table (bit_offset indicates position)
+#call_recording_url string Path to recorded .wav files
+#call_recording_server_url string URL to the recording server
+
+ });
+ $error = $cdr->insert;
+ if ($error) {
+ dbh->rollback;
+ die "Error inserting cdr: $error";
+ }
+ } #foreach $xdr
+} #foreach $custnum
+
+dbh->commit;
+
+exit;
+
+sub call_api {
+ my ($service,$method,$params) = @_;
+ my %auth_info = $auth_info ? ('auth_info' => encode_json($auth_info)) : ();
+ $params ||= {};
+ print "Calling $service/$method...\n" if $verbose;
+ my ( $page, $response, %reply_headers ) = https_post(
+ 'host' => $host,
+ 'port' => $port,
+ 'path' => '/rest/'.$service.'/'.$method.'/',
+ 'args' => [ %auth_info, 'params' => encode_json($params) ],
+ );
+ return decode_json($page) if $response eq '200 OK';
+ if ($response =~ /^500/) {
+ my $error = decode_json($page);
+ die "Server returned error during $service/$method: ".$error->{'faultstring'}
+ if $error->{'faultcode'};
+ }
+ die "Bad response from server during $service/$method: $response";
+}
+
+
-----------------------------------------------------------------------
Summary of changes:
FS/bin/freeside-cdr-portaone-import | 203 +++++++++++++++++++++++++++++++++++
1 file changed, 203 insertions(+)
create mode 100644 FS/bin/freeside-cdr-portaone-import
More information about the freeside-commits
mailing list