[freeside-commits] branch FREESIDE_4_BRANCH updated. 63c0876fa02e8872a0d1163242d0475f921b26ce

Jonathan Prykop jonathan at 420.am
Tue Jan 3 17:46:15 PST 2017


The branch, FREESIDE_4_BRANCH has been updated
       via  63c0876fa02e8872a0d1163242d0475f921b26ce (commit)
       via  d7590a1aa7d3accb0329f0579ffa167b3a67e184 (commit)
       via  bbaaff786a27e08d7f8f1e13e5c24d07c31c605f (commit)
       via  0b6e0f716c1c3ec2ddae6a3a9001808ea7761e1a (commit)
       via  c20f91e20a83c338e36a859ca07c73b94a3a38de (commit)
       via  691b4de7c0a7d0e39bebe2ea7fadea5fb21b1539 (commit)
       via  b6151684a227ad8018de4fdc661fcb61d00675a7 (commit)
       via  130db7caee3817758dcf3906d2e975e4eff0e466 (commit)
       via  e5a0dc30df71b2a9490480c0333d88cec688c035 (commit)
       via  439ec8b79f5a430e4850d4620287e7774b4fb1e4 (commit)
       via  6bf73cace9d0cc630f54ec8b2cdb2d0fb6132cf5 (commit)
       via  b1fac2d6e401888a496ca1ac48fae1f46498333b (commit)
       via  998d7aa2fd1394d70a4043fac9eac8fbec8cc41d (commit)
       via  32d2dd6ca5cbf5fd557583502211cbc4c036e22e (commit)
       via  553c407c1a2162c3cc220f54a286f3c43ae5f0e7 (commit)
       via  9547ad4e7cc87b1cfe6bc9f2428da81481e18926 (commit)
       via  3d8958a36f22a88738b637b4d5583e989e91bc8e (commit)
       via  9605850e1b105d527961a0766ec05840b3d6962e (commit)
       via  e328428724f2274144fb3e33704131ba70d20016 (commit)
       via  6f2add8c2496952f0953ae066cfde3570610c98e (commit)
       via  868e9dd529dc43fd523a6883ee72ec9b22e11b90 (commit)
       via  995b2edc47e2285db5c2cd64ef0783f2cbc37ee8 (commit)
       via  5f2c60fc31d06443fb43b30f62a4d2789aad55cd (commit)
       via  9026d337602fd1f9c463de2a46db4f27c0d429f9 (commit)
       via  1185daff43389fe53ad43e84962329a63d31523e (commit)
       via  16498ac263bf5f3e90e23b866706ada768486f40 (commit)
       via  80542a7f5c52ac2f631adc82d0e4326554200793 (commit)
       via  a77a43c3e472c12a2a343d92fd96611a00a704b6 (commit)
      from  ed832a11546a4bad4961165291b025790b76b68e (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.

- Log -----------------------------------------------------------------
commit 63c0876fa02e8872a0d1163242d0475f921b26ce
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Jan 3 19:21:05 2017 -0600

    71513: Card tokenization [comment tweak]

diff --git a/FS/FS/Upgrade.pm b/FS/FS/Upgrade.pm
index 31311e9..bf79b9c 100644
--- a/FS/FS/Upgrade.pm
+++ b/FS/FS/Upgrade.pm
@@ -373,6 +373,7 @@ sub upgrade_data {
     'agent_payment_gateway' => [],
 
     #cust_main (tokenizes cards, remove paycvv from history, locations, cust_payby, etc)
+    # (handles payinfo encryption/tokenization across all relevant tables)
     'cust_main' => [],
 
     #contact -> cust_contact / prospect_contact

commit d7590a1aa7d3accb0329f0579ffa167b3a67e184
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Jan 3 19:15:22 2017 -0600

    71513: Card tokenization [test tweak, v4 only]

diff --git a/FS/t/suite/15-activate_encryption.t b/FS/t/suite/15-activate_encryption.t
index e5732f7..dec130b 100755
--- a/FS/t/suite/15-activate_encryption.t
+++ b/FS/t/suite/15-activate_encryption.t
@@ -2,7 +2,7 @@
 
 use strict;
 use FS::Test;
-use Test::More tests => 13;
+use Test::More tests => 15;
 use FS::Conf;
 use FS::UID qw( dbh );
 use DateTime;
@@ -16,6 +16,22 @@ my @tables = qw(cust_payby cust_pay_pending cust_pay cust_pay_void cust_refund);
 ### can only run on test database (company name "Freeside Test")
 like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
 
+### remove gateway overrides and set non-tokenizing default gateway
+### so that we play nicely with tokenization upgrades
+### (though really, should just get rid of cardtype overrides in test db)
+# these will just get in the way for now
+foreach my $apg ($fs->qsearch('agent_payment_gateway')) {
+  $err = $apg->delete;
+  last if $err;
+}
+ok( !$err, 'removing agent gateway overrides' ) or BAIL_OUT($err);
+
+my $bopconf = 
+'IPPay
+TESTTERMINAL';
+$conf->set('business-onlinepayment' => $bopconf);
+is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "set default gateway" ) or BAIL_OUT('');
+
 ### we need to unencrypt our test db before we can test turning it on
 
 # temporarily load all payinfo into memory

commit bbaaff786a27e08d7f8f1e13e5c24d07c31c605f
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Jan 3 17:33:26 2017 -0600

    71513: Card tokenization [banned_pay tweaks]

diff --git a/FS/FS/banned_pay.pm b/FS/FS/banned_pay.pm
index 3d51bcd..0afff33 100644
--- a/FS/FS/banned_pay.pm
+++ b/FS/FS/banned_pay.pm
@@ -4,7 +4,7 @@ use base qw( FS::otaker_Mixin FS::Record );
 use strict;
 use Digest::MD5 qw(md5_base64);
 use Digest::SHA qw( sha512_base64 );
-use FS::Record qw( qsearchs dbh );
+use FS::Record qw( qsearch qsearchs dbh );
 use FS::CurrentUser;
 
 =head1 NAME
@@ -169,6 +169,14 @@ sub ban_search {
 # Used by FS::Upgrade to migrate to a new database.
 sub _upgrade_data {  # class method
   my ($class, %opts) = @_;
+
+  die "Cannot upgrade md5 banned_pay entries"
+    if qsearch({
+      'table'     => 'banned_pay',
+      'hashref'   => {},
+      'extra_sql' => "WHERE payinfo_hash IS NULL OR payinfo_hash = '' OR payinfo_hash = 'MD5'",
+    });
+
   $class->_upgrade_otaker(%opts);
 }
 
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index e636e88..8b1a60a 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -416,6 +416,13 @@ sub realtime_bop {
   # set fields from passed cust_payby
   _bop_cust_payby_options(\%options);
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{$options{method}},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
   if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
@@ -502,16 +509,6 @@ sub realtime_bop {
   die $@ if $@;
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{$options{method}},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # check for term discount validity
   ###
 
@@ -1793,6 +1790,13 @@ sub realtime_verify_bop {
   return "No cust_payby" unless $options{'cust_payby'};
   _bop_cust_payby_options(\%options);
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{'CC'},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
   if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
@@ -1813,16 +1817,6 @@ sub realtime_verify_bop {
   die $@ if $@;
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{'CC'},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # massage data
   ###
 
@@ -2230,6 +2224,13 @@ sub realtime_tokenize {
   return '' unless $options{method} eq 'CC';
   return '' if $self->tokenized($options{payinfo}); #already tokenized
 
+  # check for banned credit card/ACH
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{'CC'},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
   ###
   # select a gateway
   ###
@@ -2257,16 +2258,6 @@ sub realtime_tokenize {
   return '' unless $supported_actions{'CC'} and grep(/^Tokenize$/,@{$supported_actions{'CC'}});
 
   ###
-  # check for banned credit card/ACH
-  ###
-
-  my $ban = FS::banned_pay->ban_search(
-    'payby'   => $bop_method2payby{'CC'},
-    'payinfo' => $options{payinfo},
-  );
-  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
-
-  ###
   # massage data
   ###
 

commit 0b6e0f716c1c3ec2ddae6a3a9001808ea7761e1a
Merge: c20f91e ed832a1
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Jan 3 18:01:05 2017 -0600

    Merge branch 'FREESIDE_4_BRANCH' of git.freeside.biz:/home/git/freeside into FREESIDE_4_BRANCH


commit c20f91e20a83c338e36a859ca07c73b94a3a38de
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Wed Dec 28 13:57:43 2016 -0600

    73085: Enable credit card/ach encryption on a live system [handling for custnum-less cust_pay_pending]

diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index 4bd3f26..5b113f8 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -5837,6 +5837,9 @@ sub queueable_upgrade {
         # window for possible conflict is practically nonexistant,
         #   but just in case...
         $record = $record->select_for_update;
+        if (!$record->custnum && $table eq 'cust_pay_pending') {
+          $record->set('custnum_pending',1);
+        }
         my $error = $record->replace;
         die $error if $error;
       }

commit 691b4de7c0a7d0e39bebe2ea7fadea5fb21b1539
Merge: f26f465 b615168
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Wed Dec 28 14:18:29 2016 -0600

    71513: Card tokenization [project branch merge]

diff --cc FS/FS/cust_main.pm
index 2147ce1,71552b0..4bd3f26
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@@ -5789,80 -5788,9 +5796,82 @@@ sub _upgrade_data { #class metho
  
  sub queueable_upgrade {
    my $class = shift;
 +
 +  ### encryption gets turned on in _upgrade_data, above
 +
 +  eval "use FS::upgrade_journal";
 +  die $@ if $@;
 +
 +  # prior to 2013 (commit f16665c9) payinfo was stored in history if not encrypted,
 +  # clear that out before encrypting/tokenizing anything else
 +  if (!FS::upgrade_journal->is_done('clear_payinfo_history')) {
 +    foreach my $table ('cust_payby','cust_pay_pending','cust_pay','cust_pay_void','cust_refund') {
 +      my $sql = 'UPDATE h_'.$table.' SET payinfo = NULL WHERE payinfo IS NOT NULL';
 +      my $sth = dbh->prepare($sql) or die dbh->errstr;
 +      $sth->execute or die $sth->errstr;
 +    }
 +    FS::upgrade_journal->set_done('clear_payinfo_history');
 +  }
 +
 +  # encrypt old records
 +  if ($conf->exists('encryption') && !FS::upgrade_journal->is_done('encryption_check')) {
 +
 +    # allow replacement of closed cust_pay/cust_refund records
 +    local $FS::payinfo_Mixin::allow_closed_replace = 1;
 +
 +    # because it looks like nothing's changing
 +    local $FS::Record::no_update_diff = 1;
 +
 +    # commit everything immediately
 +    local $FS::UID::AutoCommit = 1;
 +
 +    # encrypt what's there
 +    foreach my $table ('cust_payby','cust_pay_pending','cust_pay','cust_pay_void','cust_refund') {
 +      my $tclass = 'FS::'.$table;
 +      my $lastrecnum = 0;
 +      my @recnums = ();
 +      while (my $recnum = _upgrade_next_recnum(dbh,$table,\$lastrecnum,\@recnums)) {
 +        my $record = $tclass->by_key($recnum);
 +        next unless $record; # small chance it's been deleted, that's ok
 +        next unless grep { $record->payby eq $_ } @FS::Record::encrypt_payby;
 +        # window for possible conflict is practically nonexistant,
 +        #   but just in case...
 +        $record = $record->select_for_update;
 +        my $error = $record->replace;
 +        die $error if $error;
 +      }
 +    }
 +
 +    FS::upgrade_journal->set_done('encryption_check');
 +  }
 +
++  # now that everything's encrypted, tokenize...
+   FS::cust_main::Billing_Realtime::token_check(@_);
  }
  
 +# not entirely false laziness w/ Billing_Realtime::_token_check_next_recnum
 +# cust_payby might get deleted while this runs
 +# not a method!
 +sub _upgrade_next_recnum {
 +  my ($dbh,$table,$lastrecnum,$recnums) = @_;
 +  my $recnum = shift @$recnums;
 +  return $recnum if $recnum;
 +  my $tclass = 'FS::'.$table;
 +  my $sql = 'SELECT '.$tclass->primary_key.
 +            ' FROM '.$table.
 +            ' WHERE '.$tclass->primary_key.' > '.$$lastrecnum.
 +            ' ORDER BY '.$tclass->primary_key.' LIMIT 500';;
 +  my $sth = $dbh->prepare($sql) or die $dbh->errstr;
 +  $sth->execute() or die $sth->errstr;
 +  my @recnums;
 +  while (my $rec = $sth->fetchrow_hashref) {
 +    push @$recnums, $rec->{$tclass->primary_key};
 +  }
 +  $sth->finish();
 +  $$lastrecnum = $$recnums[-1];
 +  return shift @$recnums;
 +}
 +
  =back
  
  =head1 BUGS

commit b6151684a227ad8018de4fdc661fcb61d00675a7
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Wed Dec 28 13:56:31 2016 -0600

    71513: Card tokenization [undid refund always sending token, skip token_check if nothing is tokenizable]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 28ed3c4..e636e88 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -1587,12 +1587,6 @@ sub realtime_refund_bop {
         $content{'name'} = $self->get('first'). ' '. $self->get('last');
       }
     }
-    if ( $cust_pay->payby eq 'CARD'
-         && !$content{'card_number'}
-         && $cust_pay->tokenized
-    ) {
-      $content{'card_token'} = $cust_pay->payinfo;
-    }
     $void->content( 'action' => 'void', %content );
     $void->test_transaction(1)
       if $conf->exists('business-onlinepayment-test_transaction');
@@ -2413,8 +2407,9 @@ sub token_check {
 
   my $cache = {}; #cache for module info
 
-  # look for a gateway that can't tokenize
+  # look for a gateway that can and can't tokenize
   my $require_tokenized = 1;
+  my $someone_tokenizing = 0;
   foreach my $gateway (
     FS::payment_gateway->all_gateways(
       'method'  => 'CC',
@@ -2426,18 +2421,26 @@ sub token_check {
       # no default gateway, no promise to tokenize
       # can just load other gateways as-needeed below
       $require_tokenized = 0;
-      last;
+      last if $someone_tokenizing;
+      next;
     }
     my $info = _token_check_gateway_info($cache,$gateway);
     die $info unless ref($info); # means it's an error message
-    unless ($info->{'can_tokenize'}) {
+    if ($info->{'can_tokenize'}) {
+      $someone_tokenizing = 1;
+    } else {
       # a configured gateway can't tokenize, that's all we need to know right now
       # can just load other gateways as-needeed below
       $require_tokenized = 0;
-      last;
+      last if $someone_tokenizing;
     }
   }
 
+  unless ($someone_tokenizing) { #no need to check, if no one can tokenize
+    warn "no gateways tokenize\n" if $debug;
+    return;
+  }
+
   warn "REQUIRE TOKENIZED" if $require_tokenized && $debug;
 
   # upgrade does not call this with autocommit turned on,

commit 130db7caee3817758dcf3906d2e975e4eff0e466
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Mon Dec 12 12:06:38 2016 -0600

    RT#71513: Card tokenization [refund test expansion]

diff --git a/FS/t/suite/14-tokenization_refund.t b/FS/t/suite/14-tokenization_refund.t
index 65202fd..1a0f840 100755
--- a/FS/t/suite/14-tokenization_refund.t
+++ b/FS/t/suite/14-tokenization_refund.t
@@ -8,7 +8,7 @@ use FS::cust_main;
 use Business::CreditCard qw(generate_last_digit);
 use DateTime;
 if ( stat('/usr/local/etc/freeside/cardfortresstest.txt') ) {
-  plan tests => 33;
+  plan tests => 66;
 } else {
   plan skip_all => 'CardFortress test encryption key is not installed.';
 }
@@ -23,10 +23,6 @@ my @bopconf;
 ### can only run on test database (company name "Freeside Test")
 like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
 
-### database might need to be upgraded before this,
-### but doesn't matter if existing records are tokenized or not,
-### this is all about testing new record creation
-
 # these will just get in the way for now
 foreach my $apg ($fs->qsearch('agent_payment_gateway')) {
   $err = $apg->delete;
@@ -62,123 +58,173 @@ gateway_password
 private_key
 /usr/local/etc/freeside/cardfortresstest.txt';
 
-# for attempting refund post-tokenization
-my $n_cust_main;
-my $n_cust_pay;
+foreach my $voiding (0,1) {
+  my $noun = $voiding ? 'void' : 'refund';
 
-foreach my $tokenizing (0,1) {
+  if ($voiding) {
+    $conf->delete('disable_void_after');
+    ok( !$conf->exists('disable_void_after'), 'set disable_void_after to produce voids' ) or BAIL_OUT('');
+  } else {
+    $conf->set('disable_void_after' => '0');
+    is( $conf->config('disable_void_after'), '0', 'set disable_void_after to produce refunds' ) or BAIL_OUT('');
+  }
 
-  my $adj = $tokenizing ? 'tokenizable' : 'non-tokenizable';
+  # for attempting refund post-tokenization
+  my $n_cust_main;
+  my $n_cust_pay;
 
-  # set payment gateway
-  $conf->set('business-onlinepayment' => $bopconf[$tokenizing]);
-  is( join("\n",$conf->config('business-onlinepayment')), $bopconf[$tokenizing], "set $adj default gateway" ) or BAIL_OUT('');
+  foreach my $tokenizing (0,1) {
+    my $adj = $tokenizing ? 'tokenizable' : 'non-tokenizable';
 
-  if ($tokenizing) {
+    # set payment gateway
+    $conf->set('business-onlinepayment' => $bopconf[$tokenizing]);
+    is( join("\n",$conf->config('business-onlinepayment')), $bopconf[$tokenizing], "set $adj $noun default gateway" ) or BAIL_OUT('');
 
-    my $n_paynum = $n_cust_pay->paynum;
+    # make sure we're upgraded, only need to do it once,
+    # use non-tokenizing gateway for speed,
+    # but doesn't matter if existing records are tokenized or not,
+    # this suite is all about testing new record creation
+    if (!$tokenizing && !$voiding) {
+      $err = system('freeside-upgrade','-q','admin');
+      ok( !$err, 'upgrade freeside' ) or BAIL_OUT('Error string: '.$!);
+    }
 
-    # refund the previous non-tokenized payment through CF
-    $err = $n_cust_main->realtime_refund_bop({
-      reasonnum => $reason->reasonnum,
-      paynum    => $n_paynum,
-      method    => 'CC',
-    });
-    ok( !$err, "run post-switch refund" ) or BAIL_OUT($err);
+    if ($tokenizing) {
 
-    # check for void record
-    my $n_cust_pay_void = $fs->qsearchs('cust_pay_void',{ paynum => $n_paynum });
-    isa_ok( $n_cust_pay_void, 'FS::cust_pay_void', 'post-switch void') or BAIL_OUT("paynum $n_paynum");
+      my $n_paynum = $n_cust_pay->paynum;
 
-    # check that void tokenized
-    ok ( $n_cust_pay_void->tokenized, "post-switch void tokenized" ) or BAIL_OUT("paynum $n_paynum");
+      # refund the previous non-tokenized payment through CF
+      $err = $n_cust_main->realtime_refund_bop({
+        reasonnum => $reason->reasonnum,
+        paynum    => $n_paynum,
+        method    => 'CC',
+      });
+      ok( !$err, "run post-switch $noun" ) or BAIL_OUT($err);
 
-    # check for no refund record
-    ok( !$fs->qsearch('cust_refund',{ source_paynum => $n_paynum }), "post-switch refund did not generate cust_refund" ) or BAIL_OUT("paynum $n_paynum");
+      my $n_cust_pay_void = $fs->qsearchs('cust_pay_void',{ paynum => $n_paynum });
+      my $n_cust_refund   = $fs->qsearchs('cust_refund',{ source_paynum => $n_paynum });
 
-  }
+      if ($voiding) {
 
-  # create customer
-  my $cust_main = $fs->new_customer($adj);
-  isa_ok ( $cust_main, 'FS::cust_main', "$adj customer" ) or BAIL_OUT('');
-
-  # insert customer
-  $err = $cust_main->insert;
-  ok( !$err, "insert $adj customer" ) or BAIL_OUT($err);
-
-  # add card
-  my $cust_payby;
-  my %card = random_card();
-  $err = $cust_main->save_cust_payby(
-    %card,
-    payment_payby => $card{'payby'},
-    auto => 1,
-    saved_cust_payby => \$cust_payby
-  );
-  ok( !$err, "save $adj card" ) or BAIL_OUT($err);
+        # check for void record
+        isa_ok( $n_cust_pay_void, 'FS::cust_pay_void', 'post-switch void') or BAIL_OUT("paynum $n_paynum");
 
-  # retrieve card
-  isa_ok ( $cust_payby, 'FS::cust_payby', "$adj card" ) or BAIL_OUT('');
+        # check that void tokenized
+        ok ( $n_cust_pay_void->tokenized, "post-switch void tokenized" ) or BAIL_OUT("paynum $n_paynum");
 
-  # check that card tokenized or not
-  if ($tokenizing) {
-    ok( $cust_payby->tokenized, 'new cust card tokenized' ) or BAIL_OUT('');
-  } else {
-    ok( !$cust_payby->tokenized, 'new cust card not tokenized' ) or BAIL_OUT('');
-  }
+        # check for no refund record
+        ok( !$n_cust_refund, "post-switch void did not generate cust_refund" ) or BAIL_OUT("paynum $n_paynum");
+
+      } else {
+
+        # check for refund record
+        isa_ok( $n_cust_refund, 'FS::cust_refund', 'post-switch refund') or BAIL_OUT("paynum $n_paynum");
+
+        # check that refund tokenized
+        ok ( $n_cust_refund->tokenized, "post-switch refund tokenized" ) or BAIL_OUT("paynum $n_paynum");
+
+        # check for no refund record
+        ok( !$n_cust_pay_void, "post-switch refund did not generate cust_pay_void" ) or BAIL_OUT("paynum $n_paynum");
+
+      }
 
-  # run a payment
-  $err = $cust_main->realtime_cust_payby( amount => '1.00' );
-  ok( !$err, "run $adj payment" ) or BAIL_OUT($err);
+    }
 
-  # get the payment
-  my $cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum }); 
-  isa_ok ( $cust_pay, 'FS::cust_pay', "$adj payment" ) or BAIL_OUT('');
+    # create customer
+    my $cust_main = $fs->new_customer($adj.'X'.$noun);
+    isa_ok ( $cust_main, 'FS::cust_main', "$adj $noun customer" ) or BAIL_OUT('');
+
+    # insert customer
+    $err = $cust_main->insert;
+    ok( !$err, "insert $adj $noun customer" ) or BAIL_OUT($err);
+
+    # add card
+    my $cust_payby;
+    my %card = random_card();
+    $err = $cust_main->save_cust_payby(
+      %card,
+      payment_payby => $card{'payby'},
+      auto => 1,
+      saved_cust_payby => \$cust_payby
+    );
+    ok( !$err, "save $adj $noun card" ) or BAIL_OUT($err);
+
+    # retrieve card
+    isa_ok ( $cust_payby, 'FS::cust_payby', "$adj $noun card" ) or BAIL_OUT('');
+
+    # check that card tokenized or not
+    if ($tokenizing) {
+      ok( $cust_payby->tokenized, "new $noun cust card tokenized" ) or BAIL_OUT('');
+    } else {
+      ok( !$cust_payby->tokenized, "new $noun cust card not tokenized" ) or BAIL_OUT('');
+    }
 
-  # refund the payment
-  $err = $cust_main->realtime_refund_bop({
-    reasonnum => $reason->reasonnum,
-    paynum    => $cust_pay->paynum,
-    method    => 'CC',
-  });
-  ok( !$err, "run $adj refund" ) or BAIL_OUT($err);
+    # run a payment
+    $err = $cust_main->realtime_cust_payby( amount => '1.00' );
+    ok( !$err, "run $adj $noun payment" ) or BAIL_OUT($err);
 
-  unless ($tokenizing) {
+    # get the payment
+    my $cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum }); 
+    isa_ok ( $cust_pay, 'FS::cust_pay', "$adj $noun payment" ) or BAIL_OUT('');
 
-    # run a second payment, to refund after switch
-    $err = $cust_main->realtime_cust_payby( amount => '2.00' );
-    ok( !$err, "run $adj second payment" ) or BAIL_OUT($err);
+    # refund the payment
+    $err = $cust_main->realtime_refund_bop({
+      reasonnum => $reason->reasonnum,
+      paynum    => $cust_pay->paynum,
+      method    => 'CC',
+    });
+    ok( !$err, "run $adj $noun" ) or BAIL_OUT($err);
+
+    unless ($tokenizing) {
+
+      # run a second payment, to refund after switch
+      $err = $cust_main->realtime_cust_payby( amount => '2.00' );
+      ok( !$err, "run $adj $noun second payment" ) or BAIL_OUT($err);
     
-    # get the second payment
-    $n_cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum, paid => '2.00' });
-    isa_ok ( $n_cust_pay, 'FS::cust_pay', "$adj second payment" ) or BAIL_OUT('');
+      # get the second payment
+      $n_cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum, paid => '2.00' });
+      isa_ok ( $n_cust_pay, 'FS::cust_pay', "$adj $noun second payment" ) or BAIL_OUT('');
 
-    $n_cust_main = $cust_main;
+      $n_cust_main = $cust_main;
 
-  }
+    }
 
-  #check that all transactions tokenized or not
-  foreach my $table (qw(cust_pay_pending cust_pay cust_pay_void)) {
-    foreach my $record ($fs->qsearch($table,{ custnum => $cust_main->custnum })) {
-      if ($tokenizing) {
-        $err = "record not tokenized: $table ".$record->get($record->primary_key)
-          unless $record->tokenized;
-      } else {
-        $err = "record tokenized: $table ".$record->get($record->primary_key)
-          if $record->tokenized;
+    #check that all transactions tokenized or not
+    foreach my $table (qw(cust_pay_pending cust_pay cust_pay_void cust_refund)) {
+      foreach my $record ($fs->qsearch($table,{ custnum => $cust_main->custnum })) {
+        if ($tokenizing) {
+          $err = "record not tokenized: $table ".$record->get($record->primary_key)
+            unless $record->tokenized;
+        } else {
+          $err = "record tokenized: $table ".$record->get($record->primary_key)
+            if $record->tokenized;
+        }
+        last if $err;
       }
-      last if $err;
     }
-  }
-  ok( !$err, "$adj transaction token check" ) or BAIL_OUT($err);
+    ok( !$err, "$adj transaction token check" ) or BAIL_OUT($err);
+
+    if ($voiding) {
+
+      #make sure we voided
+      ok( $fs->qsearch('cust_pay_void',{ custnum => $cust_main->custnum}), "$adj $noun record found" ) or BAIL_OUT('');
+
+      #make sure we didn't generate refund records
+      ok( !$fs->qsearch('cust_refund',{ custnum => $cust_main->custnum}), "$adj $noun did not generate cust_refund" ) or BAIL_OUT('');
 
-  #make sure we voided
-  ok( $fs->qsearch('cust_pay_void',{ custnum => $cust_main->custnum}), "$adj refund voided" ) or BAIL_OUT('');
+    } else {
+
+      #make sure we refunded
+      ok( $fs->qsearch('cust_refund',{ custnum => $cust_main->custnum}), "$adj $noun record found" ) or BAIL_OUT('');
+
+      #make sure we didn't generate void records
+      ok( !$fs->qsearch('cust_pay_void',{ custnum => $cust_main->custnum}), "$adj $noun did not generate cust_pay_void" ) or BAIL_OUT('');
+
+    }
 
-  #make sure we didn't generate refund records
-  ok( !$fs->qsearch('cust_refund',{ custnum => $cust_main->custnum}), "$adj refund did not generate cust_refund" ) or BAIL_OUT('');
+  } #end of tokenizing or not
 
-};
+} # end of voiding or not
 
 exit;
 

commit e5a0dc30df71b2a9490480c0333d88cec688c035
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Fri Dec 9 12:45:01 2016 -0600

    71513: Card tokenization [refund testing & bug fixes]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 0ef423b..28ed3c4 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -1454,9 +1454,10 @@ sub realtime_refund_bop {
       ( $gatewaynum, $processor, $auth, $order_number ) = ( $2, $3, $4, $6 );
     }
 
+    my $payment_gateway;
     if ( $gatewaynum ) { #gateway for the payment to be refunded
 
-      my $payment_gateway =
+      $payment_gateway =
         qsearchs('payment_gateway', { 'gatewaynum' => $gatewaynum } );
       die "payment gateway $gatewaynum not found"
         unless $payment_gateway;
@@ -1470,7 +1471,7 @@ sub realtime_refund_bop {
     } else { #try the default gateway
 
       my $conf_processor;
-      my $payment_gateway =
+      $payment_gateway =
         $self->agent->payment_gateway('method' => $options{method});
 
       ( $conf_processor, $login, $password, $namespace ) =
@@ -1487,8 +1488,27 @@ sub realtime_refund_bop {
         unless ($processor eq $conf_processor)
             || (($conf_processor eq 'CardFortress') && ($processor eq $bop_options{'gateway'}));
 
+      $processor = $conf_processor;
+
     }
 
+    # if gateway has switched to CardFortress but token_check hasn't run yet,
+    # tokenize just this record now, so that token gets passed/set appropriately
+    if ($cust_pay->payby eq 'CARD' && !$cust_pay->tokenized) {
+      my %tokenopts = (
+        'payment_gateway' => $payment_gateway,
+        'method'          => 'CC',
+        'payinfo'         => $cust_pay->payinfo,
+        'paydate'         => $cust_pay->paydate,
+      );
+      my $error = $self->realtime_tokenize(\%tokenopts); # no-op unless gateway can tokenize
+      if ($self->tokenized($tokenopts{'payinfo'})) { # implies no error
+        warn "  tokenizing cust_pay\n" if $DEBUG > 1;
+        $cust_pay->payinfo($tokenopts{'payinfo'});
+        $error = $cust_pay->replace;
+      }
+      return $error if $error;
+    }
 
   } else { # didn't specify a paynum, so look for agent gateway overrides
            # like a normal transaction 
@@ -1567,6 +1587,12 @@ sub realtime_refund_bop {
         $content{'name'} = $self->get('first'). ' '. $self->get('last');
       }
     }
+    if ( $cust_pay->payby eq 'CARD'
+         && !$content{'card_number'}
+         && $cust_pay->tokenized
+    ) {
+      $content{'card_token'} = $cust_pay->payinfo;
+    }
     $void->content( 'action' => 'void', %content );
     $void->test_transaction(1)
       if $conf->exists('business-onlinepayment-test_transaction');
diff --git a/FS/t/suite/14-tokenization_refund.t b/FS/t/suite/14-tokenization_refund.t
new file mode 100755
index 0000000..65202fd
--- /dev/null
+++ b/FS/t/suite/14-tokenization_refund.t
@@ -0,0 +1,200 @@
+#!/usr/bin/perl
+
+use strict;
+use FS::Test;
+use Test::More;
+use FS::Conf;
+use FS::cust_main;
+use Business::CreditCard qw(generate_last_digit);
+use DateTime;
+if ( stat('/usr/local/etc/freeside/cardfortresstest.txt') ) {
+  plan tests => 33;
+} else {
+  plan skip_all => 'CardFortress test encryption key is not installed.';
+}
+
+#local $FS::cust_main::Billing_Realtime::DEBUG = 2;
+
+my $fs = FS::Test->new( user => 'admin' );
+my $conf = FS::Conf->new;
+my $err;
+my @bopconf;
+
+### can only run on test database (company name "Freeside Test")
+like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
+
+### database might need to be upgraded before this,
+### but doesn't matter if existing records are tokenized or not,
+### this is all about testing new record creation
+
+# these will just get in the way for now
+foreach my $apg ($fs->qsearch('agent_payment_gateway')) {
+  $err = $apg->delete;
+  last if $err;
+}
+ok( !$err, 'removing agent gateway overrides' ) or BAIL_OUT($err);
+
+# will need this
+my $reason = FS::reason->new_or_existing(
+  reason => 'Token Test',
+  type   => 'Refund',
+  class  => 'F',
+);
+isa_ok ( $reason, 'FS::reason', "refund reason" ) or BAIL_OUT('');
+
+# non-tokenizing gateway
+push @bopconf,
+'IPPay
+TESTTERMINAL';
+
+# tokenizing gateway
+push @bopconf,
+'CardFortress
+cardfortresstest
+(TEST54)
+Normal Authorization
+gateway
+IPPay
+gateway_login
+TESTTERMINAL
+gateway_password
+
+private_key
+/usr/local/etc/freeside/cardfortresstest.txt';
+
+# for attempting refund post-tokenization
+my $n_cust_main;
+my $n_cust_pay;
+
+foreach my $tokenizing (0,1) {
+
+  my $adj = $tokenizing ? 'tokenizable' : 'non-tokenizable';
+
+  # set payment gateway
+  $conf->set('business-onlinepayment' => $bopconf[$tokenizing]);
+  is( join("\n",$conf->config('business-onlinepayment')), $bopconf[$tokenizing], "set $adj default gateway" ) or BAIL_OUT('');
+
+  if ($tokenizing) {
+
+    my $n_paynum = $n_cust_pay->paynum;
+
+    # refund the previous non-tokenized payment through CF
+    $err = $n_cust_main->realtime_refund_bop({
+      reasonnum => $reason->reasonnum,
+      paynum    => $n_paynum,
+      method    => 'CC',
+    });
+    ok( !$err, "run post-switch refund" ) or BAIL_OUT($err);
+
+    # check for void record
+    my $n_cust_pay_void = $fs->qsearchs('cust_pay_void',{ paynum => $n_paynum });
+    isa_ok( $n_cust_pay_void, 'FS::cust_pay_void', 'post-switch void') or BAIL_OUT("paynum $n_paynum");
+
+    # check that void tokenized
+    ok ( $n_cust_pay_void->tokenized, "post-switch void tokenized" ) or BAIL_OUT("paynum $n_paynum");
+
+    # check for no refund record
+    ok( !$fs->qsearch('cust_refund',{ source_paynum => $n_paynum }), "post-switch refund did not generate cust_refund" ) or BAIL_OUT("paynum $n_paynum");
+
+  }
+
+  # create customer
+  my $cust_main = $fs->new_customer($adj);
+  isa_ok ( $cust_main, 'FS::cust_main', "$adj customer" ) or BAIL_OUT('');
+
+  # insert customer
+  $err = $cust_main->insert;
+  ok( !$err, "insert $adj customer" ) or BAIL_OUT($err);
+
+  # add card
+  my $cust_payby;
+  my %card = random_card();
+  $err = $cust_main->save_cust_payby(
+    %card,
+    payment_payby => $card{'payby'},
+    auto => 1,
+    saved_cust_payby => \$cust_payby
+  );
+  ok( !$err, "save $adj card" ) or BAIL_OUT($err);
+
+  # retrieve card
+  isa_ok ( $cust_payby, 'FS::cust_payby', "$adj card" ) or BAIL_OUT('');
+
+  # check that card tokenized or not
+  if ($tokenizing) {
+    ok( $cust_payby->tokenized, 'new cust card tokenized' ) or BAIL_OUT('');
+  } else {
+    ok( !$cust_payby->tokenized, 'new cust card not tokenized' ) or BAIL_OUT('');
+  }
+
+  # run a payment
+  $err = $cust_main->realtime_cust_payby( amount => '1.00' );
+  ok( !$err, "run $adj payment" ) or BAIL_OUT($err);
+
+  # get the payment
+  my $cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum }); 
+  isa_ok ( $cust_pay, 'FS::cust_pay', "$adj payment" ) or BAIL_OUT('');
+
+  # refund the payment
+  $err = $cust_main->realtime_refund_bop({
+    reasonnum => $reason->reasonnum,
+    paynum    => $cust_pay->paynum,
+    method    => 'CC',
+  });
+  ok( !$err, "run $adj refund" ) or BAIL_OUT($err);
+
+  unless ($tokenizing) {
+
+    # run a second payment, to refund after switch
+    $err = $cust_main->realtime_cust_payby( amount => '2.00' );
+    ok( !$err, "run $adj second payment" ) or BAIL_OUT($err);
+    
+    # get the second payment
+    $n_cust_pay = $fs->qsearchs('cust_pay',{ custnum => $cust_main->custnum, paid => '2.00' });
+    isa_ok ( $n_cust_pay, 'FS::cust_pay', "$adj second payment" ) or BAIL_OUT('');
+
+    $n_cust_main = $cust_main;
+
+  }
+
+  #check that all transactions tokenized or not
+  foreach my $table (qw(cust_pay_pending cust_pay cust_pay_void)) {
+    foreach my $record ($fs->qsearch($table,{ custnum => $cust_main->custnum })) {
+      if ($tokenizing) {
+        $err = "record not tokenized: $table ".$record->get($record->primary_key)
+          unless $record->tokenized;
+      } else {
+        $err = "record tokenized: $table ".$record->get($record->primary_key)
+          if $record->tokenized;
+      }
+      last if $err;
+    }
+  }
+  ok( !$err, "$adj transaction token check" ) or BAIL_OUT($err);
+
+  #make sure we voided
+  ok( $fs->qsearch('cust_pay_void',{ custnum => $cust_main->custnum}), "$adj refund voided" ) or BAIL_OUT('');
+
+  #make sure we didn't generate refund records
+  ok( !$fs->qsearch('cust_refund',{ custnum => $cust_main->custnum}), "$adj refund did not generate cust_refund" ) or BAIL_OUT('');
+
+};
+
+exit;
+
+sub random_card {
+  my $payinfo = '4111' . join('', map { int(rand(10)) } 1 .. 11);
+  $payinfo .= generate_last_digit($payinfo);
+  my $paydate = DateTime->now
+                ->add('years' => 1)
+                ->truncate(to => 'month')
+                ->strftime('%F');
+  return ( 'payby'    => 'CARD',
+           'payinfo'  => $payinfo,
+           'paydate'  => $paydate,
+           'payname'  => 'Tokenize Me',
+  );
+}
+
+1;
+

commit 439ec8b79f5a430e4850d4620287e7774b4fb1e4
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Mon Dec 5 10:06:10 2016 -0600

    71513: Card tokenization [v4 test db handling]

diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
index 0a965aa..edb0f38 100755
--- a/FS/t/suite/13-tokenization.t
+++ b/FS/t/suite/13-tokenization.t
@@ -8,7 +8,7 @@ use FS::cust_main;
 use Business::CreditCard qw(generate_last_digit);
 use DateTime;
 if ( stat('/usr/local/etc/freeside/cardfortresstest.txt') ) {
-  plan tests => 18;
+  plan tests => 20;
 } else {
   plan skip_all => 'CardFortress test encryption key is not installed.';
 }
@@ -23,7 +23,23 @@ my $bopconf;
 
 like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
 
-# test db no longer contains cardtype overrides
+# some pre-upgrade cleanup, upgrade will fail if these are still configured
+foreach my $cust_main ( $fs->qsearch('cust_main') ) {
+  my @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+  if (@count > 1) {
+    note("DELETING CARDTYPE GATEWAYS");
+    foreach my $apg (@count) {
+      $err = $apg->delete if $apg->cardtype;
+      last if $err;
+    }
+    @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+    if (@count > 1) {
+      $err = "Still found ". at count." gateways for custnum ".$cust_main->custnum;
+      last;
+    }
+  }
+}
+ok( !$err, "remove obsolete payment gateways" ) or BAIL_OUT($err);
 
 $bopconf = 
 'IPPay
@@ -121,6 +137,24 @@ private_key
 $conf->set('business-onlinepayment' => $bopconf);
 is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting tokenizable default gateway" ) or BAIL_OUT('');
 
+foreach my $pg ($fs->qsearch('payment_gateway')) {
+  unless ($pg->gateway_module eq 'CardFortress') {
+    note('UPGRADING NON-CF PAYMENT GATEWAY');
+    my %pgopts = (
+      gateway          => $pg->gateway_module,
+      gateway_login    => $pg->gateway_username,
+      gateway_password => $pg->gateway_password,
+      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
+    );
+    $pg->gateway_module('CardFortress');
+    $pg->gateway_username('cardfortresstest');
+    $pg->gateway_password('(TEST54)');
+    $err = $pg->replace(\%pgopts);
+    last if $err;
+  }
+}
+ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
+
 # create a payment using a non-tokenized card. this should immediately
 # trigger tokenization.
 ok( $payby[1]->payby eq 'CARD' && ! $payby[1]->tokenized,

commit 6bf73cace9d0cc630f54ec8b2cdb2d0fb6132cf5
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Mon Dec 5 08:53:13 2016 -0600

    71513: Card tokenization [minor test tweaks]

diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
index b4d204f..0a965aa 100755
--- a/FS/t/suite/13-tokenization.t
+++ b/FS/t/suite/13-tokenization.t
@@ -137,27 +137,8 @@ ok( $payment[1]->tokenized, "payment is tokenized" );
 $payby[1] = $payby[1]->replace_old;
 ok( $payby[1]->tokenized, "card is now tokenized" );
 
-# test db doesn't have this
-#foreach my $pg ($fs->qsearch('payment_gateway')) {
-#  unless ($pg->gateway_module eq 'CardFortress') {
-#    note('UPGRADING NON-CF PAYMENT GATEWAY');
-#    my %pgopts = (
-#      gateway          => $pg->gateway_module,
-#      gateway_login    => $pg->gateway_username,
-#      gateway_password => $pg->gateway_password,
-#      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
-#    );
-#    $pg->gateway_module('CardFortress');
-#    $pg->gateway_username('cardfortresstest');
-#    $pg->gateway_password('(TEST54)');
-#    $err = $pg->replace(\%pgopts);
-#    last if $err;
-#  }
-#}
-#ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
-
 # invoke the part of freeside-upgrade that tokenizes
-FS::cust_main->queueable_upgrade( quiet => 1 );
+FS::cust_main->queueable_upgrade();
 #$err = system('freeside-upgrade','admin');
 #ok( !$err, 'tokenizable upgrade' ) or BAIL_OUT('Error string: '.$!);
 

commit b1fac2d6e401888a496ca1ac48fae1f46498333b
Author: Mark Wells <mark at freeside.biz>
Date:   Sun Dec 4 23:13:04 2016 -0800

    rework card tokenization test

diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
index 9a3ef3f..b4d204f 100755
--- a/FS/t/suite/13-tokenization.t
+++ b/FS/t/suite/13-tokenization.t
@@ -1,36 +1,29 @@
 #!/usr/bin/perl
 
+use strict;
 use FS::Test;
-use Test::More tests => 9;
+use Test::More;
 use FS::Conf;
+use FS::cust_main;
+use Business::CreditCard qw(generate_last_digit);
+use DateTime;
+if ( stat('/usr/local/etc/freeside/cardfortresstest.txt') ) {
+  plan tests => 18;
+} else {
+  plan skip_all => 'CardFortress test encryption key is not installed.';
+}
 
 ### can only run on test database (company name "Freeside Test")
 ### will run upgrade, which uses lots of prints & warns beyond regular test output
 
 my $fs = FS::Test->new( user => 'admin' );
-my $conf = new_ok('FS::Conf');
+my $conf = FS::Conf->new;
 my $err;
 my $bopconf;
 
 like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
 
-# some pre-upgrade cleanup, upgrade will fail if these are still configured
-foreach my $cust_main ( $fs->qsearch('cust_main') ) {
-  my @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
-  if (@count > 1) {
-    note("DELETING CARDTYPE GATEWAYS");
-    foreach my $apg (@count) {
-      $err = $apg->delete if $apg->cardtype;
-      last if $err;
-    }
-    @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
-    if (@count > 1) {
-      $err = "Still found ". at count." gateways for custnum ".$cust_main->custnum;
-      last;
-    }
-  }
-}
-ok( !$err, "remove obsolete payment gateways" ) or BAIL_OUT($err);
+# test db no longer contains cardtype overrides
 
 $bopconf = 
 'IPPay
@@ -78,9 +71,40 @@ foreach my $cust_pay ( $fs->qsearch('cust_pay',{ payby => 'CARD' }) ) {
 }
 ok( !$err, "create some refunds and voids" ) or BAIL_OUT($err);
 
+# also, just to test behavior in this case, create a record for an aborted
+# verification payment. this will have no customer number.
+
+my $pending_failed = FS::cust_pay_pending->new({
+  'custnum_pending' => 1,
+  'paid'    => '1.00',
+  '_date'   => time - 86400,
+  random_card(),
+  'status'  => 'failed',
+  'statustext' => 'Tokenization upgrade test',
+});
+$err = $pending_failed->insert;
+ok( !$err, "create a failed payment attempt" ) or BAIL_OUT($err);
+
+# find two stored credit cards.
+my @cust = map { FS::cust_main->by_key($_) } (10, 12);
+my @payby = map { ($_->cust_payby)[0] } @cust;
+my @payment;
+
+ok( $payby[0]->payby eq 'CARD' && !$payby[0]->tokenized,
+  "first customer has a non-tokenized card"
+  ) or BAIL_OUT();
+
+$err = $cust[0]->realtime_cust_payby(amount => '2.00');
+ok( !$err, "create a payment through IPPay" )
+  or BAIL_OUT($err);
+$payment[0] = $fs->qsearchs('cust_pay', { custnum => $cust[0]->custnum,
+                                     paid => '2.00' })
+  or BAIL_OUT("can't find payment record");
+
 $err = system('freeside-upgrade','admin');
 ok( !$err, 'initial upgrade' ) or BAIL_OUT('Error string: '.$!);
 
+# switch to CardFortress
 $bopconf =
 'CardFortress
 cardfortresstest
@@ -97,26 +121,78 @@ private_key
 $conf->set('business-onlinepayment' => $bopconf);
 is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting tokenizable default gateway" ) or BAIL_OUT('');
 
-foreach my $pg ($fs->qsearch('payment_gateway')) {
-  unless ($pg->gateway_module eq 'CardFortress') {
-    note('UPGRADING NON-CF PAYMENT GATEWAY');
-    my %pgopts = (
-      gateway          => $pg->gateway_module,
-      gateway_login    => $pg->gateway_username,
-      gateway_password => $pg->gateway_password,
-      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
-    );
-    $pg->gateway_module('CardFortress');
-    $pg->gateway_username('cardfortresstest');
-    $pg->gateway_password('(TEST54)');
-    $err = $pg->replace(\%pgopts);
-    last if $err;
-  }
+# create a payment using a non-tokenized card. this should immediately
+# trigger tokenization.
+ok( $payby[1]->payby eq 'CARD' && ! $payby[1]->tokenized,
+  "second customer has a non-tokenized card"
+  ) or BAIL_OUT();
+
+$err = $cust[1]->realtime_cust_payby(amount => '3.00');
+ok( !$err, "tokenize a card when it's first used for payment" )
+  or BAIL_OUT($err);
+$payment[1] = $fs->qsearchs('cust_pay', { custnum => $cust[1]->custnum,
+                                     paid => '3.00' })
+  or BAIL_OUT("can't find payment record");
+ok( $payment[1]->tokenized, "payment is tokenized" );
+$payby[1] = $payby[1]->replace_old;
+ok( $payby[1]->tokenized, "card is now tokenized" );
+
+# test db doesn't have this
+#foreach my $pg ($fs->qsearch('payment_gateway')) {
+#  unless ($pg->gateway_module eq 'CardFortress') {
+#    note('UPGRADING NON-CF PAYMENT GATEWAY');
+#    my %pgopts = (
+#      gateway          => $pg->gateway_module,
+#      gateway_login    => $pg->gateway_username,
+#      gateway_password => $pg->gateway_password,
+#      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
+#    );
+#    $pg->gateway_module('CardFortress');
+#    $pg->gateway_username('cardfortresstest');
+#    $pg->gateway_password('(TEST54)');
+#    $err = $pg->replace(\%pgopts);
+#    last if $err;
+#  }
+#}
+#ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
+
+# invoke the part of freeside-upgrade that tokenizes
+FS::cust_main->queueable_upgrade( quiet => 1 );
+#$err = system('freeside-upgrade','admin');
+#ok( !$err, 'tokenizable upgrade' ) or BAIL_OUT('Error string: '.$!);
+
+$payby[0] = $payby[0]->replace_old;
+ok( $payby[0]->tokenized, "old card was tokenized during upgrade" );
+$payment[0] = $payment[0]->replace_old;
+ok( $payment[0]->tokenized, "old payment was tokenized during upgrade" );
+ok( ($payment[0]->cust_pay_pending)[0]->tokenized, "old cust_pay_pending was tokenized during upgrade" );
+
+$pending_failed = $pending_failed->replace_old;
+ok( $pending_failed->tokenized, "cust_pay_pending with no customer was tokenized" );
+
+# add a new payment card to one customer
+$payby[2] = FS::cust_payby->new({
+  custnum => $cust[0]->custnum,
+  random_card(),
+});
+$err = $payby[2]->insert;
+ok( !$err, "new card was saved" );
+ok($payby[2]->tokenized, "new card is tokenized" );
+
+sub random_card {
+  my $payinfo = '4111' . join('', map { int(rand(10)) } 1 .. 11);
+  $payinfo .= generate_last_digit($payinfo);
+  my $paydate = DateTime->now
+                ->add('years' => 1)
+                ->truncate(to => 'month')
+                ->strftime('%F');
+  return ( 'payby'    => 'CARD',
+           'payinfo'  => $payinfo,
+           'paydate'  => $paydate,
+           'payname'  => 'Tokenize Me',
+  );
 }
-ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
 
-$err = system('freeside-upgrade','admin');
-ok( !$err, 'tokenizable upgrade' ) or BAIL_OUT('Error string: '.$!);
 
 1;
 

commit 998d7aa2fd1394d70a4043fac9eac8fbec8cc41d
Author: Mark Wells <mark at freeside.biz>
Date:   Sun Dec 4 21:35:52 2016 -0800

    minor fix for custnum-less cust_pay_pending, #71513

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 0ea94c8..0ef423b 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -2552,7 +2552,11 @@ CUSTLOOP:
         # might happen for cust_pay_pending from failed verify records,
         #   in which case we attempt tokenization without cust_main
         # everything else should absolutely have a cust_main
-        unless ($table eq 'cust_pay_pending' && $record->{'custnum_pending'}) {
+        if ($table eq 'cust_pay_pending' and !$record->custnum ) {
+          # override the usual safety check and allow the record to be
+          # updated even without a custnum.
+          $record->set('custnum_pending', 1);
+        } else {
           my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
           if ($opt{'queue'}) {
             $log->critical($error);
@@ -2634,7 +2638,7 @@ CUSTLOOP:
       warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug && !$cust_main;
 
       # if we got this far, time to mutex
-      $record = $record->select_for_update;
+      $record->select_for_update;
 
       # no clear record of name/address/etc used for transaction,
       # but will load name/phone/id from customer if run as an object method,

commit 32d2dd6ca5cbf5fd557583502211cbc4c036e22e
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Sat Dec 3 10:36:03 2016 -0600

    71513: Card tokenization [refund gateway choice]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 68431fc..0ea94c8 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -1480,10 +1480,12 @@ sub realtime_refund_bop {
       @bop_options = $payment_gateway->gatewaynum
                        ? $payment_gateway->options
                        : @{ $payment_gateway->get('options') };
+      my %bop_options = @bop_options;
 
       return "processor of payment $options{'paynum'} $processor does not".
              " match default processor $conf_processor"
-        unless $processor eq $conf_processor;
+        unless ($processor eq $conf_processor)
+            || (($conf_processor eq 'CardFortress') && ($processor eq $bop_options{'gateway'}));
 
     }
 
@@ -1492,9 +1494,7 @@ sub realtime_refund_bop {
            # like a normal transaction 
  
     my $payment_gateway =
-      $self->agent->payment_gateway( 'method'  => $options{method},
-                                     #'payinfo' => $payinfo,
-                                   );
+      $self->agent->payment_gateway( 'method'  => $options{method} );
     my( $processor, $login, $password, $namespace ) =
       map { my $method = "gateway_$_"; $payment_gateway->$method }
         qw( module username password namespace );
@@ -1627,18 +1627,22 @@ sub realtime_refund_bop {
     if length($payip);
 
   my $payinfo = '';
+  my $paymask = ''; # for refund record
   if ( $options{method} eq 'CC' ) {
 
     if ( $cust_pay ) {
       $content{card_number} = $payinfo = $cust_pay->payinfo;
+      $paymask = $cust_pay->paymask;
       (exists($options{'paydate'}) ? $options{'paydate'} : $cust_pay->paydate)
         =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/ &&
         ($content{expiration} = "$2/$1");  # where available
     } else {
-      $content{card_number} = $payinfo = $self->payinfo;
-      (exists($options{'paydate'}) ? $options{'paydate'} : $self->paydate)
-        =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
-      $content{expiration} = "$2/$1";
+      # this really needs a better cleanup
+      die "Refund without paynum not supported";
+#      $content{card_number} = $payinfo = $self->payinfo;
+#      (exists($options{'paydate'}) ? $options{'paydate'} : $self->paydate)
+#        =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
+#      $content{expiration} = "$2/$1";
     }
 
   } elsif ( $options{method} eq 'ECHECK' ) {
@@ -1702,6 +1706,7 @@ sub realtime_refund_bop {
     '_date'    => '',
     'payby'    => $bop_method2payby{$options{method}},
     'payinfo'  => $payinfo,
+    'paymask'  => $paymask,
     'reasonnum'     => $options{'reasonnum'},
     'gatewaynum'    => $gatewaynum, # may be null
     'processor'     => $processor,
@@ -2442,7 +2447,7 @@ CUSTLOOP:
       }
 
       # only load gateway if we need to, and only need to load it once
-      my $payment_gateway ||= $cust_main->_payment_gateway({
+      $payment_gateway ||= $cust_main->_payment_gateway({
         'method'  => 'CC',
         'conf'    => $conf,
         'nofatal' => 1, # handle lack of gateway smoothly below
@@ -2542,15 +2547,72 @@ CUSTLOOP:
         $dbh->commit or die $dbh->errstr; # commit log message
       }
 
-      # don't use customer agent gateway here, use the gatewaynum specified by the record
-      my $gateway = FS::payment_gateway->by_key_or_default( 
-        'method'     => 'CC',
-        'conf'       => $conf,
-        'nofatal'    => 1,
-        'gatewaynum' => $record->gatewaynum || '',
-      );
+      my $cust_main = $record->cust_main;
+      if (!$cust_main) {
+        # might happen for cust_pay_pending from failed verify records,
+        #   in which case we attempt tokenization without cust_main
+        # everything else should absolutely have a cust_main
+        unless ($table eq 'cust_pay_pending' && $record->{'custnum_pending'}) {
+          my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
+          if ($opt{'queue'}) {
+            $log->critical($error);
+            $dbh->commit or die $dbh->errstr; # commit log message
+            next;
+          }
+          $dbh->rollback if $oldAutoCommit;
+          die $error;
+        }
+      }
+
+      my $gateway;
+
+      # use the gatewaynum specified by the record if possible
+      $gateway = FS::payment_gateway->by_key_with_namespace(
+        'gatewaynum' => $record->gatewaynum,
+      ) if $record->gateway;
+
+      # otherwise use the cust agent gateway if possible (which realtime_refund_bop would do)
+      # otherwise just use default gateway
+      unless ($gateway) {
+
+        $gateway = $cust_main 
+                 ? $cust_main->agent->payment_gateway
+                 : FS::payment_gateway->default_gateway;
+
+        # check for processor mismatch
+        unless ($table eq 'cust_pay_pending') { # has no processor table
+          if (my $processor = $record->processor) {
+
+            my $conf_processor = $gateway->gateway_module;
+            my %bop_options = $gateway->gatewaynum
+                            ? $gateway->options
+                            : @{ $gateway->get('options') };
+
+            # this is the same standard used by realtime_refund_bop
+            unless (
+              ($processor eq $conf_processor) ||
+              (($conf_processor eq 'CardFortress') && ($processor eq $bop_options{'gateway'}))
+            ) {
+
+              # processors don't match, so refund already cannot be run on this object,
+              # regardless of what we do now...
+              # but unless we gotta tokenize everything, just leave well enough alone
+              unless ($require_tokenized) {
+                warn "Skipping mismatched processor for $table ".$record->get($record->primary_key) if $debug;
+                next;
+              }
+              ### no error--we'll tokenize using the new gateway, just to remove stored payinfo,
+              ### because refunds are already impossible for this record, anyway
+
+            } # end processor mismatch
+
+          } # end record has processor
+        } # end not cust_pay_pending
+
+      }
+
+      # means no default gateway, no promise to tokenize, can skip
       unless ($gateway) {
-        # means no default gateway, no promise to tokenize, can skip
         warn "Skipping missing gateway for $table ".$record->get($record->primary_key) if $debug;
         next;
       }
@@ -2569,24 +2631,7 @@ CUSTLOOP:
         next;
       }
 
-      my $cust_main = $record->cust_main;
-      if (!$cust_main) {
-        # might happen for cust_pay_pending from failed verify records,
-        #   in which case we attempt tokenization without cust_main
-        # everything else should absolutely have a cust_main
-        if ($table eq 'cust_pay_pending' && $record->{'custnum_pending'}) {
-          warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug;
-        } else {
-          my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
-          if ($opt{'queue'}) {
-            $log->critical($error);
-            $dbh->commit or die $dbh->errstr; # commit log message
-            next;
-          }
-          $dbh->rollback if $oldAutoCommit;
-          die $error;
-        }
-      }
+      warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug && !$cust_main;
 
       # if we got this far, time to mutex
       $record = $record->select_for_update;
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index be37568..3a51022 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -195,8 +195,6 @@ sub payinfo_check {
   FS::payby->can_payby($self->table, $self->payby)
     or return "Illegal payby: ". $self->payby;
 
-  my $conf = new FS::Conf;
-
   if ( $self->payby eq 'CARD' && ! $self->is_encrypted($self->payinfo) ) {
 
     my $payinfo = $self->payinfo;
diff --git a/FS/FS/payinfo_transaction_Mixin.pm b/FS/FS/payinfo_transaction_Mixin.pm
index c27d049..1b5a0cd 100644
--- a/FS/FS/payinfo_transaction_Mixin.pm
+++ b/FS/FS/payinfo_transaction_Mixin.pm
@@ -102,8 +102,6 @@ auth, and order_number) as well as payby and payinfo
 sub payinfo_check {
   my $self = shift;
 
-  my $conf = new FS::Conf;
-
   $self->SUPER::payinfo_check()
   || $self->ut_numbern('gatewaynum')
   # not ut_foreign_keyn, it causes upgrades to fail
diff --git a/FS/FS/payment_gateway.pm b/FS/FS/payment_gateway.pm
index 170d37a..3500bf9 100644
--- a/FS/FS/payment_gateway.pm
+++ b/FS/FS/payment_gateway.pm
@@ -385,6 +385,23 @@ sub default_gateway {
   return $payment_gateway;
 }
 
+=item by_key_with_namespace GATEWAYNUM
+
+Like usual by_key, but makes sure namespace is set,
+and dies if not found.
+
+=cut
+
+sub by_key_with_namespace {
+  my $self = shift;
+  my $payment_gateway = $self->by_key(@_);
+  die "payment_gateway not found"
+    unless $payment_gateway;
+  $payment_gateway->gateway_namespace('Business::OnlinePayment')
+    unless $payment_gateway->gateway_namespace;
+  return $payment_gateway;
+}
+
 =item by_key_or_default OPTIONS
 
 Either returns the gateway specified by option gatewaynum, or the default gateway.
@@ -399,13 +416,7 @@ sub by_key_or_default {
   my ($self,%options) = @_;
 
   if ($options{'gatewaynum'}) {
-    my $payment_gateway = $self->by_key($options{'gatewaynum'});
-    # regardless of nofatal, which is only meant for handling lack of default gateway
-    die "payment_gateway ".$options{'gatewaynum'}." not found"
-      unless $payment_gateway;
-    $payment_gateway->gateway_namespace('Business::OnlinePayment')
-      unless $payment_gateway->gateway_namespace;
-    return $payment_gateway;
+    return $self->by_key_with_namespace($options{'gatewaynum'});
   } else {
     return $self->default_gateway(%options);
   }

commit 553c407c1a2162c3cc220f54a286f3c43ae5f0e7
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Wed Nov 30 10:12:04 2016 -0600

    71513: Card tokenization [removed unneeded code]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 35c4f0d..68431fc 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -427,8 +427,6 @@ sub realtime_bop {
       $token_error = $options{'cust_payby'}->replace;
       return $token_error if $token_error;
     }
-    return "Cannot tokenize card info"
-      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ### 
@@ -1777,8 +1775,6 @@ sub realtime_verify_bop {
     return $token_error if $token_error;
     #important that we not replace cust_payby here,
     #because cust_payby->replace uses realtime_verify_bop!
-    return "Cannot tokenize card info"
-      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ###

commit 9547ad4e7cc87b1cfe6bc9f2428da81481e18926
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Wed Nov 30 05:45:29 2016 -0600

    71513: Card tokenization [bug fixes, removed cardtype/taxclass override ui]

diff --git a/FS/FS/agent.pm b/FS/FS/agent.pm
index b97e9b9..e70b971 100644
--- a/FS/FS/agent.pm
+++ b/FS/FS/agent.pm
@@ -239,8 +239,7 @@ sub ticketing_queue {
 
 Returns a payment gateway object (see L<FS::payment_gateway>) for this agent.
 
-Currently available options are I<nofatal>, I<method>, I<thirdparty>,
-<conf> and I<load_gatewaynum>.
+Currently available options are I<nofatal>, I<method>, I<thirdparty> and I<conf>.
 
 If I<nofatal> is set, and no gateway is available, then the empty string
 will be returned instead of throwing a fatal exception.
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index fb0c010..35c4f0d 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -113,7 +113,6 @@ I<depend_jobnum> allows payment capture to unlock export jobs
 =cut
 
 # Currently only used by ClientAPI
-# NOT 4.x COMPATIBLE (see below)
 sub realtime_collect {
   my( $self, %options ) = @_;
 
@@ -127,10 +126,6 @@ sub realtime_collect {
   $options{amount} = $self->balance unless exists( $options{amount} );
   return '' unless $options{amount} > 0;
 
-  #### NOT 4.x COMPATIBLE
-  $options{method} = FS::payby->payby2bop($self->payby)
-    unless exists( $options{method} );
-
   return $self->realtime_bop({%options});
 
 }
diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
index 1b654ad..9a3ef3f 100755
--- a/FS/t/suite/13-tokenization.t
+++ b/FS/t/suite/13-tokenization.t
@@ -1,7 +1,7 @@
 #!/usr/bin/perl
 
 use FS::Test;
-use Test::More tests => 8;
+use Test::More tests => 9;
 use FS::Conf;
 
 ### can only run on test database (company name "Freeside Test")
@@ -38,6 +38,46 @@ TESTTERMINAL';
 $conf->set('business-onlinepayment' => $bopconf);
 is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting first default gateway" ) or BAIL_OUT('');
 
+# generate a few void/refund records for upgrading
+my $counter = 20;
+foreach my $cust_pay ( $fs->qsearch('cust_pay',{ payby => 'CARD' }) ) {
+  if ($counter % 2) {
+    $err = $cust_pay->void('Testing');
+    $err = "Voiding: $err" if $err;
+  } else {
+    # from realtime_refund_bop, just the important bits    
+    while ( $cust_pay->unapplied < $cust_pay->paid ) {
+      my @cust_bill_pay = $cust_pay->cust_bill_pay;
+      last unless @cust_bill_pay;
+      my $cust_bill_pay = pop @cust_bill_pay;
+      $err = $cust_bill_pay->delete;
+      $err = "Refund unapply: $err" if $err;
+      last if $err;
+    }
+    last if $err;
+    my $cust_refund = new FS::cust_refund ( {
+      'custnum'  => $cust_pay->cust_main->custnum,
+      'paynum'   => $cust_pay->paynum,
+      'source_paynum' => $cust_pay->paynum,
+      'refund'   => $cust_pay->paid,
+      '_date'    => '',
+      'payby'    => $cust_pay->payby,
+      'payinfo'  => $cust_pay->payinfo,
+      'reason'     => 'Testing',
+      'gatewaynum'    => $cust_pay->gatewaynum,
+      'processor'     => $cust_pay->payment_gateway ? $cust_pay->payment_gateway->processor : '',
+      'auth'          => $cust_pay->auth,
+      'order_number'  => $cust_pay->order_number,
+    } );
+    $err = $cust_refund->insert( reason_type => 'Refund' );
+    $err = "Refunding: $err" if $err;
+  }
+  last if $err;
+  $counter -= 1;
+  last unless $counter > 0;
+}
+ok( !$err, "create some refunds and voids" ) or BAIL_OUT($err);
+
 $err = system('freeside-upgrade','admin');
 ok( !$err, 'initial upgrade' ) or BAIL_OUT('Error string: '.$!);
 
diff --git a/httemplate/browse/agent.cgi b/httemplate/browse/agent.cgi
index 3fe68c1..751e8b6 100755
--- a/httemplate/browse/agent.cgi
+++ b/httemplate/browse/agent.cgi
@@ -42,7 +42,7 @@ full offerings (via their type).<BR><BR>
     <TH CLASS="grid" BGCOLOR="#cccccc">Currencies</TH>
 % } 
 
-  <TH CLASS="grid" BGCOLOR="#cccccc"><FONT SIZE=-1>Payment Gateway Overrides</FONT></TH>
+  <TH CLASS="grid" BGCOLOR="#cccccc"><FONT SIZE=-1>Payment Gateway Override</FONT></TH>
   <TH CLASS="grid" BGCOLOR="#cccccc"><FONT SIZE=-1>Configuration Overrides</FONT></TH>
 </TR>
 
@@ -331,32 +331,24 @@ Unused
 % } 
 
 %       ##
-%       # payment gateway overrides
+%       # payment gateway override
 %       ##
 
         <TD CLASS="inv" BGCOLOR="<% $bgcolor %>">
           <TABLE CLASS="inv" CELLSPACING=0 CELLPADDING=0>
-% foreach my $override (
-%                 # sort { }  want taxclass-full stuff first?  and default cards (empty cardtype)
-%                 qsearch('agent_payment_gateway', { 'agentnum' => $agent->agentnum } )
-%               ) {
-%            
-
+% my $gw_override = qsearchs('agent_payment_gateway', { 'agentnum' => $agent->agentnum } );
+% if ($gw_override) {
               <TR>
                 <TD> 
-                  <% $override->cardtype || 'Default' %> to <% $override->payment_gateway->gateway_module %> (<% $override->payment_gateway->gateway_username %>)
-                  <% $override->taxclass
-                        ? ' for '. $override->taxclass. ' only'
-                        : ''
-                  %>
-                  <FONT SIZE=-1><A HREF="javascript:areyousure('delete this payment gateway override', '<%$p%>misc/delete-agent_payment_gateway.cgi?<% $override->agentgatewaynum %>')">(delete)</A></FONT>
+                  <% $gw_override->payment_gateway->gateway_module %> (<% $gw_override->payment_gateway->gateway_username %>)
+                  <FONT SIZE=-1><A HREF="javascript:areyousure('delete this payment gateway override', '<%$p%>misc/delete-agent_payment_gateway.cgi?<% $gw_override->agentgatewaynum %>')">(delete)</A></FONT>
                 </TD>
               </TR>
-% } 
-
+% } else {
             <TR>
               <TD><FONT SIZE=-1><A HREF="<%$p%>edit/agent_payment_gateway.html?agentnum=<% $agent->agentnum %>">(add override)</A></FONT></TD>
             </TR>
+% }
           </TABLE>
         </TD>
 
diff --git a/httemplate/edit/agent_payment_gateway.html b/httemplate/edit/agent_payment_gateway.html
index 41a9f3e..753bc76 100644
--- a/httemplate/edit/agent_payment_gateway.html
+++ b/httemplate/edit/agent_payment_gateway.html
@@ -1,6 +1,6 @@
 <% include("/elements/header.html","$action payment gateway override for ". $agent->agent,  menubar(
   #'View all payment gateways' => $p. 'browse/payment_gateway.html',
-  'View all agents' => $p. 'browse/agent.html',
+  'View all agents' => $p. 'browse/agent.cgi',
 )) %>
 
 <% include('/elements/error.html') %>
@@ -20,32 +20,6 @@ Use gateway <SELECT NAME="gatewaynum">
 </SELECT>
 <BR><BR>
 
-for <SELECT NAME="cardtype" MULTIPLE>
-% foreach my $cardtype (
-%  "",
-%  "VISA card",
-%  "MasterCard",
-%  "Discover card",
-%  "American Express card",
-%  "Diner's Club/Carte Blanche",
-%  "enRoute",
-%  "JCB",
-%  "BankCard",
-%  "Switch",
-%  "Solo",
-%  'ACH',
-%  'PayPal',
-%) { 
-
-  <OPTION VALUE="<% $cardtype %>"><% $cardtype || '(Default fallback)' %>
-% } 
-
-</SELECT>
-<BR><BR>
-
-(optional) when invoice contains only items of taxclass <INPUT TYPE="text" NAME="taxclass">
-<BR><BR>
-
 <INPUT TYPE="submit" VALUE="Add gateway override">
 </FORM>
 
diff --git a/httemplate/edit/process/agent_payment_gateway.html b/httemplate/edit/process/agent_payment_gateway.html
index 5b5fd94..c9789cf 100644
--- a/httemplate/edit/process/agent_payment_gateway.html
+++ b/httemplate/edit/process/agent_payment_gateway.html
@@ -10,20 +10,13 @@ die "agentnum $1 not found" unless $agent;
 
 #my $old
 
-my @new = map {
-                my $cardtype = $_;
-                new FS::agent_payment_gateway {
+my $new = new FS::agent_payment_gateway {
                   ( map { $_ => scalar($cgi->param($_)) }
                                     fields('agent_payment_gateway')
                   ),
-                  'cardtype' => $cardtype,
                 };
-              }
-              $cgi->param('cardtype');
 
-foreach my $new (@new) {
-  my $error = $new->insert;
-  die $error if $error;
-}
+my $error = $new->insert;
+die $error if $error;
 
 </%init>

commit 3d8958a36f22a88738b637b4d5583e989e91bc8e
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 29 04:21:46 2016 -0600

    71513: Card tokenization [upgrade implemented]

diff --git a/FS/FS/Cron/cleanup.pm b/FS/FS/Cron/cleanup.pm
index 6ec4013..9d0c067 100644
--- a/FS/FS/Cron/cleanup.pm
+++ b/FS/FS/Cron/cleanup.pm
@@ -8,12 +8,26 @@ use FS::Record qw( qsearch );
 
 # start janitor jobs
 sub cleanup {
-# fix locations that are missing coordinates
+  my %opt = @_;
+
+  # fix locations that are missing coordinates
   my $job = FS::queue->new({
       'job'     => 'FS::cust_location::process_set_coord',
       'status'  => 'new'
   });
   $job->insert('_JOB');
+
+  # check card number tokenization
+  $job = FS::queue->new({
+      'job'     => 'FS::cust_main::Billing_Realtime::token_check',
+      'status'  => 'new'
+  });
+  $job->insert(
+    %opt,
+    'queue' => 1,
+    'daily' => 1,
+  );
+
 }
 
 sub cleanup_before_backup {
diff --git a/FS/FS/Upgrade.pm b/FS/FS/Upgrade.pm
index 7fbbbaa..31311e9 100644
--- a/FS/FS/Upgrade.pm
+++ b/FS/FS/Upgrade.pm
@@ -368,7 +368,11 @@ sub upgrade_data {
     #fix whitespace - before cust_main
     'cust_location' => [],
 
-    #cust_main (remove paycvv from history, locations, cust_payby, etc)
+    # need before cust_main tokenization upgrade,
+    # blocks tokenization upgrade if deprecated features still in use
+    'agent_payment_gateway' => [],
+
+    #cust_main (tokenizes cards, remove paycvv from history, locations, cust_payby, etc)
     'cust_main' => [],
 
     #contact -> cust_contact / prospect_contact
@@ -396,10 +400,6 @@ sub upgrade_data {
     #duplicate history records
     'h_cust_svc'  => [],
 
-    # need before transaction tables, 
-    # blocks tokenization upgrade if deprecated features still in use
-    'agent_payment_gateway' => [],
-
     #populate cust_pay.otaker
     'cust_pay'    => [],
 
diff --git a/FS/FS/agent.pm b/FS/FS/agent.pm
index 8aa78c2..b97e9b9 100644
--- a/FS/FS/agent.pm
+++ b/FS/FS/agent.pm
@@ -9,6 +9,7 @@ use FS::cust_main;
 use FS::cust_pkg;
 use FS::reg_code;
 use FS::agent_payment_gateway;
+use FS::payment_gateway;
 use FS::TicketSystem;
 use FS::Conf;
 
@@ -253,12 +254,7 @@ the business-onlinepayment-ach gateway will be returned if available.
 If I<thirdparty> is set and the I<method> is PAYPAL, the defined paypal
 gateway will be returned.
 
-If I<load_gatewaynum> exists, then either the specified gateway or the
-default gateway will be returned.  Agent overrides are ignored, and this can
-safely be called as a class method if this option is specified.  Not
-compatible with I<thirdparty>.
-
-Exsisting I<$conf> may be passed for efficiency.
+Exisisting I<$conf> may be passed for efficiency.
 
 =cut
 
@@ -268,8 +264,8 @@ Exsisting I<$conf> may be passed for efficiency.
 sub payment_gateway {
   my ( $self, %options ) = @_;
   
+  $options{'conf'} ||= new FS::Conf;
   my $conf = $options{'conf'};
-  $conf ||= new FS::Conf;
 
   if ( $options{thirdparty} ) {
 
@@ -299,72 +295,12 @@ sub payment_gateway {
     }
   }
 
-  my ($override, $payment_gateway);
-  if (exists $options{'load_gatewaynum'}) { # no agent overrides if this opt is in use
-    if ($options{'load_gatewaynum'}) {
-      $payment_gateway = qsearchs('payment_gateway', { gatewaynumnum => $options{'load_gatewaynum'} } );
-      # always fatal
-      die "Could not load payment gateway ".$options{'load_gatewaynum'} unless $payment_gateway;
-    } # else use default, loaded below
-  } else {
-    $override = qsearchs('agent_payment_gateway', { agentnum => $self->agentnum } );
-  }
-
-  if ( $override ) { #use a payment gateway override
-
-    $payment_gateway = $override->payment_gateway;
-
-    $payment_gateway->gateway_namespace('Business::OnlinePayment')
-      unless $payment_gateway->gateway_namespace;
-
-  } elsif (!$payment_gateway) { #use the standard settings from the config
-
-    # the standard settings from the config could be moved to a null agent
-    # agent_payment_gateway referenced payment_gateway
-
-    # remember, this block might be run as a class method if false load_gatewaynum exists
+  my $override = qsearchs('agent_payment_gateway', { agentnum => $self->agentnum } );
 
-    unless ( $conf->exists('business-onlinepayment') ) {
-      if ( $options{'nofatal'} ) {
-        return '';
-      } else {
-        die "Real-time processing not enabled\n";
-      }
-    }
-
-    #load up config
-    my $bop_config = 'business-onlinepayment';
-    $bop_config .= '-ach'
-      if ( $options{method}
-           && $options{method} =~ /^(ECHECK|CHEK)$/
-           && $conf->exists($bop_config. '-ach')
-         );
-    my ( $processor, $login, $password, $action, @bop_options ) =
-      $conf->config($bop_config);
-    $action ||= 'normal authorization';
-    pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
-    die "No real-time processor is enabled - ".
-        "did you set the business-onlinepayment configuration value?\n"
-      unless $processor;
-
-    $payment_gateway = new FS::payment_gateway;
-
-    $payment_gateway->gateway_namespace( $conf->config('business-onlinepayment-namespace') ||
-                                 'Business::OnlinePayment');
-    $payment_gateway->gateway_module($processor);
-    $payment_gateway->gateway_username($login);
-    $payment_gateway->gateway_password($password);
-    $payment_gateway->gateway_action($action);
-    $payment_gateway->set('options', [ @bop_options ]);
-
-  }
-
-  unless ( $payment_gateway->gateway_namespace ) {
-    $payment_gateway->gateway_namespace(
-      scalar($conf->config('business-onlinepayment-namespace'))
-      || 'Business::OnlinePayment'
-    );
-  }
+  my $payment_gateway = FS::payment_gateway->by_key_or_default(
+    gatewaynum => $override ? $override->gatewaynum : '',
+    %options,
+  );
 
   $payment_gateway;
 }
diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index a2c0ee8..71552b0 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -5786,6 +5786,11 @@ sub _upgrade_data { #class method
 
 }
 
+sub queueable_upgrade {
+  my $class = shift;
+  FS::cust_main::Billing_Realtime::token_check(@_);
+}
+
 =back
 
 =head1 BUGS
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 183a7e6..fb0c010 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -14,6 +14,7 @@ use FS::cust_pay_pending;
 use FS::cust_bill_pay;
 use FS::cust_refund;
 use FS::banned_pay;
+use FS::payment_gateway;
 
 $realtime_bop_decline_quiet = 0;
 
@@ -2296,7 +2297,7 @@ sub realtime_tokenize {
     'type'           => 'CC',
     _bop_auth(\%options),          
     'action'         => 'Tokenize',
-    'description'    => $options{'description'}
+    'description'    => $options{'description'},
     %$bop_content,
     %content, #after
   );
@@ -2346,7 +2347,7 @@ sub tokenized {
   FS::cust_pay->tokenized($payinfo);
 }
 
-=item token_check
+=item token_check [ quiet => 1, queue => 1, daily => 1 ]
 
 NOT A METHOD.  Acts on all customers.  Placed here because it makes
 use of module-internal methods, and to keep everything that uses
@@ -2355,74 +2356,138 @@ Billing::OnlinePayment all in one place.
 Tokenizes all tokenizable card numbers from payinfo in cust_payby and 
 CARD transactions in cust_pay_pending, cust_pay, cust_pay_void and cust_refund.
 
-If all configured gateways have the ability to tokenize, then detection of
-an untokenizable record will cause a fatal error.
+If the I<queue> flag is set, newly tokenized records will be immediately
+committed, regardless of AutoCommit, so as to release the mutex on the record.
+
+If all configured gateways have the ability to tokenize, detection of an 
+untokenizable record will cause a fatal error.  However, if the I<queue> flag 
+is set, this will instead cause a critical error to be recorded in the log, 
+and any other tokenizable records will still be committed.
+
+If the I<daily> flag is also set, detection of existing untokenized records will 
+record a critical error in the system log (because they should have never appeared 
+in the first place.)  Tokenization will still be attempted.
+
+If any configured gateways do NOT have the ability to tokenize, or if a
+default gateway is not configured, then untokenized records are not considered 
+a threat, and no critical errors will be generated in the log.
 
 =cut
 
 sub token_check {
-  # no input, acts on all customers
+  #acts on all customers
+  my %opt = @_;
+  my $debug = !$opt{'quiet'} || $DEBUG;
 
-  eval "use FS::Cursor";  
-  return "Error initializing FS::Cursor: ".$@ if $@;
+  warn "token_check called with opts\n".Dumper(\%opt) if $debug;
 
-  my $dbh = dbh;
+  # force some explicitness when invoking this method
+  die "token_check must run with queue flag if run with daily flag"
+    if $opt{'daily'} && !$opt{'queue'};
+
+  my $conf = FS::Conf->new;
+
+  my $log = FS::Log->new('FS::cust_main::Billing_Realtime::token_check');
 
-  # get list of all gateways in table (not counting default gateway)
   my $cache = {}; #cache for module info
-  my $sth = $dbh->prepare('SELECT DISTINCT gatewaynum FROM payment_gateway')
-    or die $dbh->errstr;
-  $sth->execute or die $sth->errstr;
-  my @gatewaynums;
-  while (my $row = $sth->fetchrow_hashref) {
-    push(@gatewaynums,$row->{'gatewaynum'});
-  }
-  $sth->finish;
 
   # look for a gateway that can't tokenize
-  my $disallow_untokenized = 1;
-  foreach my $gatewaynum ('', at gatewaynums) {
-    my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum, nofatal => 1 );
-    if (!$gateway) { # already died if $gatewaynum
+  my $require_tokenized = 1;
+  foreach my $gateway (
+    FS::payment_gateway->all_gateways(
+      'method'  => 'CC',
+      'conf'    => $conf,
+      'nofatal' => 1,
+    )
+  ) {
+    if (!$gateway) {
       # no default gateway, no promise to tokenize
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
+      $require_tokenized = 0;
       last;
     }
     my $info = _token_check_gateway_info($cache,$gateway);
-    return $info unless ref($info); # means it's an error message
+    die $info unless ref($info); # means it's an error message
     unless ($info->{'can_tokenize'}) {
       # a configured gateway can't tokenize, that's all we need to know right now
       # can just load other gateways as-needeed below
-      $disallow_untokenized = 0;
+      $require_tokenized = 0;
       last;
     }
   }
 
+  warn "REQUIRE TOKENIZED" if $require_tokenized && $debug;
+
+  # upgrade does not call this with autocommit turned on,
+  # and autocommit will be ignored if opt queue is set,
+  # but might as well be thorough...
   my $oldAutoCommit = $FS::UID::AutoCommit;
   local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  # for retrieving data in chunks
+  my $step = 500;
+  my $offset = 0;
 
   ### Tokenize cust_payby
 
-  my $cust_search = FS::Cursor->new({ table => 'cust_main' },$dbh);
-  while (my $cust_main = $cust_search->fetch) {
+  my @recnums;
+
+CUSTLOOP:
+  while (my $custnum = _token_check_next_recnum($dbh,'cust_main',$step,\$offset,\@recnums)) {
+    my $cust_main = FS::cust_main->by_key($custnum);
+    my $payment_gateway;
     foreach my $cust_payby ($cust_main->cust_payby('CARD','DCRD')) {
-      next if $cust_payby->tokenized;
-      # load gateway first, just so we can cache it
-      my $payment_gateway = $cust_main->_payment_gateway({
-        'nofatal' => 1, # handle error smoothly below
+
+      # see if it's already tokenized
+      if ($cust_payby->tokenized) {
+        warn "cust_payby ".$cust_payby->get($cust_payby->primary_key)." already tokenized" if $debug;
+        next;
+      }
+
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->critical("Untokenized card number detected in cust_payby ".$cust_payby->custpaybynum);
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
+
+      # only load gateway if we need to, and only need to load it once
+      my $payment_gateway ||= $cust_main->_payment_gateway({
+        'method'  => 'CC',
+        'conf'    => $conf,
+        'nofatal' => 1, # handle lack of gateway smoothly below
       });
       unless ($payment_gateway) {
         # no reason to have untokenized card numbers saved if no gateway,
-        #   but only fatal if we expected everyone to tokenize card numbers
-        next unless $disallow_untokenized;
-        $cust_search->DESTROY;
+        #   but only a problem if we expected everyone to tokenize card numbers
+        unless ($require_tokenized) {
+          warn "Skipping cust_payby for cust_main ".$cust_main->custnum.", no payment gateway" if $debug;
+          next CUSTLOOP; # can skip rest of customer
+        }
+        my $error = "No gateway found for custnum ".$cust_main->custnum;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit error message
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for custnum ".$cust_main->custnum;
+        die $error;
       }
+
       my $info = _token_check_gateway_info($cache,$payment_gateway);
+      unless (ref($info)) {
+        # only throws error if Business::OnlinePayment won't load,
+        #   which is just cause to abort this whole process, even if queue
+        $dbh->rollback if $oldAutoCommit;
+        die $info; # error message
+      }
       # no fail here--a configured gateway can't tokenize, so be it
-      next unless ref($info) && $info->{'can_tokenize'};
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping ".$cust_main->custnum." cannot tokenize" if $debug;
+        next;
+      }
+
+      # time to tokenize
+      $cust_payby = $cust_payby->select_for_update;
       my %tokenopts = (
         'payment_gateway' => $payment_gateway,
         'cust_payby'      => $cust_payby,
@@ -2434,11 +2499,20 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $cust_search->DESTROY;
+        $error = "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next; # not next CUSTLOOP, want to record error for every cust_payby
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED cust_payby ".$cust_payby->get($cust_payby->primary_key) if $debug;
     }
+    warn "cust_payby upgraded for custnum ".$cust_main->custnum if $debug;
+
   }
 
   ### Tokenize/mask transaction tables
@@ -2449,50 +2523,83 @@ sub token_check {
   # grep assistance:
   #   $cust_pay_pending->replace, $cust_pay->replace, $cust_pay_void->replace, $cust_refund->replace all run here
   foreach my $table ( qw(cust_pay_pending cust_pay cust_pay_void cust_refund) ) {
-    my $search = FS::Cursor->new({
-      table     => $table,
-      hashref   => { 'payby' => 'CARD' },
-    },$dbh);
-    while (my $record = $search->fetch) {
-      next if $record->tokenized;
-      next if !$record->payinfo; #shouldn't happen, but at least it's not a card number
-      next if $record->payinfo =~ /N\/A/; # ??? Not sure why we do this, but it's not a card number
+    warn "Checking $table" if $debug;
+
+    # FS::Cursor does not seem to work over multiple commits (gives cursor not found errors)
+    # loading only record ids, then loading individual records one at a time
+    my $tclass = 'FS::'.$table;
+    $offset = 0;
+    @recnums = ();
+
+    while (my $recnum = _token_check_next_recnum($dbh,$table,$step,\$offset,\@recnums)) {
+      my $record = $tclass->by_key($recnum);
+      if (FS::cust_main::Billing_Realtime->tokenized($record->payinfo)) {
+        warn "Skipping tokenized record for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if (!$record->payinfo) { #shouldn't happen, but at least it's not a card number
+        warn "Skipping blank payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+      if ($record->payinfo =~ /N\/A/) { # ??? Not sure why we do this, but it's not a card number
+        warn "Skipping NA payinfo for $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
+
+      if ($require_tokenized && $opt{'daily'}) {
+        $log->critical("Untokenized card number detected in $table ".$record->get($record->primary_key));
+        $dbh->commit or die $dbh->errstr; # commit log message
+      }
 
       # don't use customer agent gateway here, use the gatewaynum specified by the record
-      my $gatewaynum = $record->gatewaynum || '';
-      my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum );
-      unless ($gateway) { # already died if $gatewaynum
-        # only fatal if we expected everyone to tokenize
-        next unless $disallow_untokenized;
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return "No gateway found for $table ".$record->get($record->primary_key);
+      my $gateway = FS::payment_gateway->by_key_or_default( 
+        'method'     => 'CC',
+        'conf'       => $conf,
+        'nofatal'    => 1,
+        'gatewaynum' => $record->gatewaynum || '',
+      );
+      unless ($gateway) {
+        # means no default gateway, no promise to tokenize, can skip
+        warn "Skipping missing gateway for $table ".$record->get($record->primary_key) if $debug;
+        next;
       }
+
       my $info = _token_check_gateway_info($cache,$gateway);
       unless (ref($info)) {
         # only throws error if Business::OnlinePayment won't load,
-        #   which is just cause to abort this whole process
-        $search->DESTROY;
+        #   which is just cause to abort this whole process, even if queue
         $dbh->rollback if $oldAutoCommit;
-        return $info; # error message
+        die $info; # error message
       }
 
       # a configured gateway can't tokenize, move along
-      next unless $info->{'can_tokenize'};
+      unless ($info->{'can_tokenize'}) {
+        warn "Skipping, cannot tokenize $table ".$record->get($record->primary_key) if $debug;
+        next;
+      }
 
       my $cust_main = $record->cust_main;
-      unless ($cust_main || (
+      if (!$cust_main) {
         # might happen for cust_pay_pending from failed verify records,
         #   in which case we attempt tokenization without cust_main
         # everything else should absolutely have a cust_main
-        $table eq 'cust_pay_pending'
-          && $record->{'custnum_pending'}
-          && !$disallow_untokenized
-      )) {
-        $search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        return "Could not load cust_main for $table ".$record->get($record->primary_key);
+        if ($table eq 'cust_pay_pending' && $record->{'custnum_pending'}) {
+          warn "ATTEMPTING GATEWAY-ONLY TOKENIZE" if $debug;
+        } else {
+          my $error = "Could not load cust_main for $table ".$record->get($record->primary_key);
+          if ($opt{'queue'}) {
+            $log->critical($error);
+            $dbh->commit or die $dbh->errstr; # commit log message
+            next;
+          }
+          $dbh->rollback if $oldAutoCommit;
+          die $error;
+        }
       }
+
+      # if we got this far, time to mutex
+      $record = $record->select_for_update;
+
       # no clear record of name/address/etc used for transaction,
       # but will load name/phone/id from customer if run as an object method,
       # so we try that if we can
@@ -2512,19 +2619,44 @@ sub token_check {
         $error ||= 'Unknown error';
       }
       if ($error) {
-        $search->DESTROY;
+        $error = "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        if ($opt{'queue'}) {
+          $log->critical($error);
+          $dbh->commit or die $dbh->errstr; # commit log message, release mutex
+          next;
+        }
         $dbh->rollback if $oldAutoCommit;
-        return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+        die $error;
       }
+      $dbh->commit or die $dbh->errstr if $opt{'queue'}; # release mutex
+      warn "TOKENIZED $table ".$record->get($record->primary_key) if $debug;
+
     } # end record loop
   } # end table loop
 
-  $dbh->commit if $oldAutoCommit;
+  $dbh->commit or die $dbh->errstr if $oldAutoCommit;
 
   return '';
 }
 
 # not a method!
+sub _token_check_next_recnum {
+  my ($dbh,$table,$step,$offset,$recnums) = @_;
+  my $recnum = shift @$recnums;
+  return $recnum if $recnum;
+  my $tclass = 'FS::'.$table;
+  my $sth = $dbh->prepare('SELECT '.$tclass->primary_key.' FROM '.$table.' ORDER BY '.$tclass->primary_key.' LIMIT '.$step.' OFFSET '.$$offset) or die $dbh->errstr;
+  $sth->execute() or die $sth->errstr;
+  my @recnums;
+  while (my $rec = $sth->fetchrow_hashref) {
+    push @$recnums, $rec->{$tclass->primary_key};
+  }
+  $sth->finish();
+  $$offset += $step;
+  return shift @$recnums;
+}
+
+# not a method!
 sub _token_check_gateway_info {
   my ($cache,$payment_gateway) = @_;
 
@@ -2562,8 +2694,6 @@ sub _token_check_gateway_info {
   $info->{'void_requires_card'} = 1
     if $transaction->info('CC_void_requires_card');
 
-  $cache->{$payment_gateway->gateway_module} = $info;
-
   return $info;
 }
 
diff --git a/FS/FS/log_context.pm b/FS/FS/log_context.pm
index 51aa79d..a41d3c8 100644
--- a/FS/FS/log_context.pm
+++ b/FS/FS/log_context.pm
@@ -11,6 +11,7 @@ my @contexts = ( qw(
   FS::cust_main::Billing_Realtime::realtime_bop
   FS::cust_main::Billing_Realtime::realtime_tokenize
   FS::cust_main::Billing_Realtime::realtime_verify_bop
+  FS::cust_main::Billing_Realtime::token_check
   FS::pay_batch::import_from_gateway
   FS::part_pkg
   FS::Misc::Geo::standardize_uscensus
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 2f50312..be37568 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -468,6 +468,7 @@ Optionally, an arbitrary payby and payinfo can be passed.
 sub tokenized {
   my $self = shift;
   my $payinfo = scalar(@_) ? shift : $self->payinfo;
+  return 0 unless $payinfo; #avoid uninitialized value error
   $payinfo =~ /^99\d{14}$/;
 }
 
diff --git a/FS/FS/payment_gateway.pm b/FS/FS/payment_gateway.pm
index afae266..170d37a 100644
--- a/FS/FS/payment_gateway.pm
+++ b/FS/FS/payment_gateway.pm
@@ -323,6 +323,108 @@ sub processor {
   }
 }
 
+=item default_gateway OPTIONS
+
+Class method.
+
+Returns default gateway (from business-onlinepayment conf) as a payment_gateway object.
+
+Accepts options
+
+conf - existing conf object
+
+nofatal - return blank instead of dying if no default gateway is configured
+
+method - if set to CHEK or ECHECK, returns object for business-onlinepayment-ach if available
+
+Before using this, be sure you wouldn't rather be using L</by_key_or_default> or,
+more likely, L<FS::agent/payment_gateway>.
+
+=cut
+
+# the standard settings from the config could be moved to a null agent
+# agent_payment_gateway referenced payment_gateway
+
+sub default_gateway {
+  my ($self,%options) = @_;
+
+  $options{'conf'} ||= new FS::Conf;
+  my $conf = $options{'conf'};
+
+  unless ( $conf->exists('business-onlinepayment') ) {
+    if ( $options{'nofatal'} ) {
+      return '';
+    } else {
+      die "Real-time processing not enabled\n";
+    }
+  }
+
+  #load up config
+  my $bop_config = 'business-onlinepayment';
+  $bop_config .= '-ach'
+    if ( $options{method}
+         && $options{method} =~ /^(ECHECK|CHEK)$/
+         && $conf->exists($bop_config. '-ach')
+       );
+  my ( $processor, $login, $password, $action, @bop_options ) =
+    $conf->config($bop_config);
+  $action ||= 'normal authorization';
+  pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
+  die "No real-time processor is enabled - ".
+      "did you set the business-onlinepayment configuration value?\n"
+    unless $processor;
+
+  my $payment_gateway = new FS::payment_gateway;
+  $payment_gateway->gateway_namespace( $conf->config('business-onlinepayment-namespace') ||
+                                       'Business::OnlinePayment');
+  $payment_gateway->gateway_module($processor);
+  $payment_gateway->gateway_username($login);
+  $payment_gateway->gateway_password($password);
+  $payment_gateway->gateway_action($action);
+  $payment_gateway->set('options', [ @bop_options ]);
+  return $payment_gateway;
+}
+
+=item by_key_or_default OPTIONS
+
+Either returns the gateway specified by option gatewaynum, or the default gateway.
+
+Accepts the same options as L</default_gateway>.
+
+Also ensures that the gateway_namespace has been set.
+
+=cut
+
+sub by_key_or_default {
+  my ($self,%options) = @_;
+
+  if ($options{'gatewaynum'}) {
+    my $payment_gateway = $self->by_key($options{'gatewaynum'});
+    # regardless of nofatal, which is only meant for handling lack of default gateway
+    die "payment_gateway ".$options{'gatewaynum'}." not found"
+      unless $payment_gateway;
+    $payment_gateway->gateway_namespace('Business::OnlinePayment')
+      unless $payment_gateway->gateway_namespace;
+    return $payment_gateway;
+  } else {
+    return $self->default_gateway(%options);
+  }
+}
+
+# if it weren't for the way gateway_namespace default is set, this method would not be necessary
+# that should really go in check() with an accompanying upgrade, so we could just use qsearch safely,
+# but currently short on time to test deeper changes...
+#
+# if no default gateway is set and nofatal is passed, first value returned is blank string
+sub all_gateways {
+  my ($self,%options) = @_;
+  my @out;
+  foreach my $gatewaynum ('',( map {$_->gatewaynum} qsearch('payment_gateway') )) {
+    push @out, $self->by_key_or_default( %options, gatewaynum => $gatewaynum );
+  }
+  return @out;
+}
+
 # _upgrade_data
 #
 # Used by FS::Upgrade to migrate to a new database.
diff --git a/FS/bin/freeside-daily b/FS/bin/freeside-daily
index ee95c14..e1463f5 100755
--- a/FS/bin/freeside-daily
+++ b/FS/bin/freeside-daily
@@ -97,7 +97,7 @@ use FS::Cron::backup qw(backup);
 backup();
 
 #except we'd rather not start cleanup jobs until the backup is done
-cleanup();
+cleanup( quiet => !$opt{'v'} );
 
 $log->info('finish');
 
diff --git a/FS/t/suite/13-tokenization.t b/FS/t/suite/13-tokenization.t
new file mode 100755
index 0000000..1b654ad
--- /dev/null
+++ b/FS/t/suite/13-tokenization.t
@@ -0,0 +1,82 @@
+#!/usr/bin/perl
+
+use FS::Test;
+use Test::More tests => 8;
+use FS::Conf;
+
+### can only run on test database (company name "Freeside Test")
+### will run upgrade, which uses lots of prints & warns beyond regular test output
+
+my $fs = FS::Test->new( user => 'admin' );
+my $conf = new_ok('FS::Conf');
+my $err;
+my $bopconf;
+
+like( $conf->config('company_name'), qr/^Freeside Test/, 'using test database' ) or BAIL_OUT('');
+
+# some pre-upgrade cleanup, upgrade will fail if these are still configured
+foreach my $cust_main ( $fs->qsearch('cust_main') ) {
+  my @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+  if (@count > 1) {
+    note("DELETING CARDTYPE GATEWAYS");
+    foreach my $apg (@count) {
+      $err = $apg->delete if $apg->cardtype;
+      last if $err;
+    }
+    @count = $fs->qsearch('agent_payment_gateway', { agentnum => $cust_main->agentnum } );
+    if (@count > 1) {
+      $err = "Still found ". at count." gateways for custnum ".$cust_main->custnum;
+      last;
+    }
+  }
+}
+ok( !$err, "remove obsolete payment gateways" ) or BAIL_OUT($err);
+
+$bopconf = 
+'IPPay
+TESTTERMINAL';
+$conf->set('business-onlinepayment' => $bopconf);
+is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting first default gateway" ) or BAIL_OUT('');
+
+$err = system('freeside-upgrade','admin');
+ok( !$err, 'initial upgrade' ) or BAIL_OUT('Error string: '.$!);
+
+$bopconf =
+'CardFortress
+cardfortresstest
+(TEST54)
+Normal Authorization
+gateway
+IPPay
+gateway_login
+TESTTERMINAL
+gateway_password
+
+private_key
+/usr/local/etc/freeside/cardfortresstest.txt';
+$conf->set('business-onlinepayment' => $bopconf);
+is( join("\n",$conf->config('business-onlinepayment')), $bopconf, "setting tokenizable default gateway" ) or BAIL_OUT('');
+
+foreach my $pg ($fs->qsearch('payment_gateway')) {
+  unless ($pg->gateway_module eq 'CardFortress') {
+    note('UPGRADING NON-CF PAYMENT GATEWAY');
+    my %pgopts = (
+      gateway          => $pg->gateway_module,
+      gateway_login    => $pg->gateway_username,
+      gateway_password => $pg->gateway_password,
+      private_key      => '/usr/local/etc/freeside/cardfortresstest.txt',
+    );
+    $pg->gateway_module('CardFortress');
+    $pg->gateway_username('cardfortresstest');
+    $pg->gateway_password('(TEST54)');
+    $err = $pg->replace(\%pgopts);
+    last if $err;
+  }
+}
+ok( !$err, "remove non-CF payment gateways" ) or BAIL_OUT($err);
+
+$err = system('freeside-upgrade','admin');
+ok( !$err, 'tokenizable upgrade' ) or BAIL_OUT('Error string: '.$!);
+
+1;
+
diff --git a/httemplate/edit/elements/edit.html b/httemplate/edit/elements/edit.html
index 35818dd..7e0eee4 100644
--- a/httemplate/edit/elements/edit.html
+++ b/httemplate/edit/elements/edit.html
@@ -247,7 +247,7 @@ Example:
   >
 
   <INPUT TYPE="hidden" NAME="svcdb" VALUE="<% $table %>">
-  <INPUT TYPE="hidden" NAME="<% $pkey %>" VALUE="<% $clone ? '' : $object->$pkey() %>">
+  <INPUT TYPE="hidden" ID="<% $pkey %>" NAME="<% $pkey %>" VALUE="<% $clone ? '' : $object->$pkey() %>">
 
   <% defined($opt{'form_init'}) 
         ? ( ref($opt{'form_init'})
diff --git a/httemplate/edit/payment_gateway.html b/httemplate/edit/payment_gateway.html
index b44b315..f9b8f24 100644
--- a/httemplate/edit/payment_gateway.html
+++ b/httemplate/edit/payment_gateway.html
@@ -22,6 +22,9 @@
 <SCRIPT TYPE="text/javascript">
   var modulesForNamespace = <% $json->encode(\%modules) %>;
   function changeNamespace() {
+    if (document.getElementById('gatewaynum').value) {
+      return true;
+    }
     var ns = document.getElementById('gateway_namespace').value;
     var select_module = document.getElementById('gateway_module');
     select_module.options.length = 0;
@@ -180,7 +183,13 @@ my $field_callback = sub {
   my ($cgi, $object, $field_hashref ) = @_;
   if ($object->gatewaynum) {
     if ( $field_hashref->{field} eq 'gateway_module' ) {
-      $field_hashref->{type} = 'fixed';
+      if ($object->gateway_namespace eq 'Business::OnlinePayment' &&
+          $object->gateway_module ne 'CardFortress'
+      ) {
+        $field_hashref->{options} = [ $object->gateway_module, 'CardFortress' ]
+      } else {
+        $field_hashref->{type} = 'fixed';
+      }
     } elsif ( $field_hashref->{field} eq 'gateway_namespace' ) {
       $field_hashref->{type} = 'fixed';
       $field_hashref->{formatted_value} = $object->namespace_description;

commit 9605850e1b105d527961a0766ec05840b3d6962e
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 29 02:46:10 2016 -0600

    Bug fix to #73185, discovered via #71513

diff --git a/FS/FS/Cron/tax_rate_update.pm b/FS/FS/Cron/tax_rate_update.pm
index b6ac63c..fec696f 100755
--- a/FS/FS/Cron/tax_rate_update.pm
+++ b/FS/FS/Cron/tax_rate_update.pm
@@ -31,7 +31,7 @@ sub tax_rate_update {
   my %opt = @_;
 
   my $oldAutoCommit = $FS::UID::AutoCommit;
-  $FS::UID::AutoCommit = 0;
+  local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
 
   my $conf = FS::Conf->new;

commit e328428724f2274144fb3e33704131ba70d20016
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 22 18:40:39 2016 -0600

    71513: Card tokenization [cust_pay_pending handling, bug fixes]

diff --git a/FS/FS/Upgrade.pm b/FS/FS/Upgrade.pm
index 9c0a230..7fbbbaa 100644
--- a/FS/FS/Upgrade.pm
+++ b/FS/FS/Upgrade.pm
@@ -49,7 +49,7 @@ sub upgrade_config {
 
   # to simplify tokenization upgrades
   die "Conf selfservice-payment_gateway no longer supported"
-    if conf->config('selfservice-payment_gateway');
+    if $conf->config('selfservice-payment_gateway');
 
   $conf->touch('payment_receipt')
     if $conf->exists('payment_receipt_email')
diff --git a/FS/FS/agent_payment_gateway.pm b/FS/FS/agent_payment_gateway.pm
index 4991c19..6a7cc06 100644
--- a/FS/FS/agent_payment_gateway.pm
+++ b/FS/FS/agent_payment_gateway.pm
@@ -1,5 +1,6 @@
 package FS::agent_payment_gateway;
 use base qw(FS::Record);
+use FS::Record qw( qsearch );
 
 use strict;
 
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 48b6ee6..183a7e6 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -223,6 +223,7 @@ sub _bop_recurring_billing {
 
 }
 
+#can run safely as class method if opt payment_gateway already exists
 sub _payment_gateway {
   my ($self, $options) = @_;
 
@@ -239,8 +240,9 @@ sub _payment_gateway {
   $options->{payment_gateway};
 }
 
+# not a method!!!
 sub _bop_auth {
-  my ($self, $options) = @_;
+  my ($options) = @_;
 
   (
     'login'    => $options->{payment_gateway}->gateway_username,
@@ -282,8 +284,9 @@ sub _bop_defaults {
 
 }
 
+# not a method!
 sub _bop_cust_payby_options {
-  my ($self,$options) = @_;
+  my ($options) = @_;
   my $cust_payby = $options->{'cust_payby'};
   if ($cust_payby) {
 
@@ -319,6 +322,8 @@ sub _bop_cust_payby_options {
   }
 }
 
+# can be called as class method,
+# but can't load default name/phone fields as class method
 sub _bop_content {
   my ($self, $options) = @_;
   my %content = ();
@@ -339,16 +344,16 @@ sub _bop_content {
       /^\s*([\w \,\.\-\']*)?\s+([\w\,\.\-\']+)\s*$/
       or return "Illegal payname $payname";
     ($payfirst, $paylast) = ($1, $2);
-  } else {
+  } elsif (ref($self)) { # can't set payname if called as class method
     $payfirst = $self->getfield('first');
     $paylast = $self->getfield('last');
     $payname = "$payfirst $paylast";
   }
 
-  $content{last_name} = $paylast;
-  $content{first_name} = $payfirst;
+  $content{last_name} = $paylast if $paylast;
+  $content{first_name} = $payfirst if $payfirst;
 
-  $content{name} = $payname;
+  $content{name} = $payname if $payname;
 
   $content{address} = $options->{'address1'};
   my $address2 = $options->{'address2'};
@@ -359,7 +364,9 @@ sub _bop_content {
   $content{zip} = $options->{'zip'};
   $content{country} = $options->{'country'};
 
-  $content{phone} = $self->daytime || $self->night;
+  # can't set phone if called as class method
+  $content{phone} = $self->daytime || $self->night
+    if ref($self);
 
   my $currency =    $conf->exists('business-onlinepayment-currency')
                  && $conf->config('business-onlinepayment-currency');
@@ -369,6 +376,7 @@ sub _bop_content {
 }
 
 # updates payinfo and cust_payby options with token from transaction
+# can be called as a class method
 sub _tokenize_card {
   my ($self,$transaction,$options) = @_;
   if ( $transaction->can('card_token') 
@@ -410,7 +418,7 @@ sub realtime_bop {
   }
 
   # set fields from passed cust_payby
-  $self->_bop_cust_payby_options(\%options);
+  _bop_cust_payby_options(\%options);
 
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
@@ -698,7 +706,7 @@ sub realtime_bop {
 
   $transaction->content(
     'type'           => $options{method},
-    $self->_bop_auth(\%options),          
+    _bop_auth(\%options),          
     'action'         => $action1,
     'description'    => $options{'description'},
     'amount'         => $options{amount},
@@ -760,7 +768,7 @@ sub realtime_bop {
       %content,
       type           => $options{method},
       action         => $action2,
-      $self->_bop_auth(\%options),          
+      _bop_auth(\%options),          
       order_number   => $ordernum,
       amount         => $options{amount},
       authorization  => $auth,
@@ -1291,7 +1299,7 @@ sub realtime_botpp_capture {
 
   $transaction->content(
     'type'           => $method,
-    $self->_bop_auth(\%options),
+    _bop_auth(\%options),
     'action'         => 'Post Authorization',
     'description'    => $options{'description'},
     'amount'         => $cust_pay_pending->paid,
@@ -1764,7 +1772,7 @@ sub realtime_verify_bop {
 
   # set fields from passed cust_payby
   return "No cust_payby" unless $options{'cust_payby'};
-  $self->_bop_cust_payby_options(\%options);
+  _bop_cust_payby_options(\%options);
 
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
@@ -1911,7 +1919,7 @@ sub realtime_verify_bop {
 
     $transaction->content(
       'type'           => 'CC',
-      $self->_bop_auth(\%options),          
+      _bop_auth(\%options),          
       'action'         => 'Authorization Only',
       'description'    => $options{'description'},
       'amount'         => '1.00',
@@ -1958,7 +1966,7 @@ sub realtime_verify_bop {
                                   );
 
       $reverse->content( 'action'        => 'Reverse Authorization',
-                         $self->_bop_auth(\%options),          
+                         _bop_auth(\%options),          
 
                          # B:OP
                          'amount'        => '1.00',
@@ -2177,8 +2185,13 @@ Otherwise, options I<method>, I<payinfo> and other cust_payby fields
 may be passed.  If options are passed as a hashref, I<payinfo>
 will be updated as appropriate in the passed hashref.
 
+Can be run as a class method if option I<payment_gateway> is passed,
+but default customer id/name/phone can't be set in that case.  This
+is really only intended for tokenizing old records on upgrade.
+
 =cut
 
+# careful--might be run as a class method
 sub realtime_tokenize {
   my $self = shift;
 
@@ -2196,7 +2209,7 @@ sub realtime_tokenize {
   }
 
   # set fields from passed cust_payby
-  $self->_bop_cust_payby_options(\%options);
+  _bop_cust_payby_options(\%options);
   return '' unless $options{method} eq 'CC';
   return '' if $self->tokenized($options{payinfo}); #already tokenized
 
@@ -2240,6 +2253,11 @@ sub realtime_tokenize {
   # massage data
   ###
 
+  ### Currently, cardfortress only keys in on card number and exp date.
+  ### We pass everything we'd pass to a normal transaction,
+  ### for ease of current and future development,
+  ### but note, when tokenizing old records, we may only have access to payinfo/paydate
+
   my $bop_content = $self->_bop_content(\%options);
   return $bop_content unless ref($bop_content);
 
@@ -2263,6 +2281,9 @@ sub realtime_tokenize {
   my $payissue       = $options{'payissue'};
   $content{issue_number} = $payissue if $payissue;
 
+  $content{customer_id} = $self->custnum
+    if ref($self);
+
   ###
   # run transaction
   ###
@@ -2273,10 +2294,9 @@ sub realtime_tokenize {
 
   $transaction->content(
     'type'           => 'CC',
-    $self->_bop_auth(\%options),          
+    _bop_auth(\%options),          
     'action'         => 'Tokenize',
-    'description'    => $options{'description'},
-    'customer_id'    => $self->custnum,
+    'description'    => $options{'description'}
     %$bop_content,
     %content, #after
   );
@@ -2314,7 +2334,9 @@ sub realtime_tokenize {
 
 Convenience wrapper for L<FS::payinfo_Mixin/tokenized>
 
-PAYINFO is required
+PAYINFO is required.
+
+Can be run as class or object method, never loads from object.
 
 =cut
 
@@ -2421,6 +2443,9 @@ sub token_check {
 
   ### Tokenize/mask transaction tables
 
+  # allow tokenization of closed cust_pay/cust_refund records
+  local $FS::payinfo_Mixin::allow_closed_replace = 1;
+
   # grep assistance:
   #   $cust_pay_pending->replace, $cust_pay->replace, $cust_pay_void->replace, $cust_refund->replace all run here
   foreach my $table ( qw(cust_pay_pending cust_pay cust_pay_void cust_refund) ) {
@@ -2456,34 +2481,35 @@ sub token_check {
       next unless $info->{'can_tokenize'};
 
       my $cust_main = $record->cust_main;
-      unless ($cust_main) {
-        # might happen for cust_pay_pending for failed verify records,
-        #   in which case it *should* already be tokenized if possible
-        #   but only get strict about it if we're expecting full tokenization
-        next if 
-          $table eq 'cust_pay_pending'
-            && $record->{'custnum_pending'}
-            && !$disallow_untokenized;
-        # XXX we currently need a $cust_main to run realtime_tokenize
-        #     even if we made it a class method, wouldn't have access to payname/etc.
-        #     fail for now, but probably could handle this better...
+      unless ($cust_main || (
+        # might happen for cust_pay_pending from failed verify records,
+        #   in which case we attempt tokenization without cust_main
         # everything else should absolutely have a cust_main
+        $table eq 'cust_pay_pending'
+          && $record->{'custnum_pending'}
+          && !$disallow_untokenized
+      )) {
         $search->DESTROY;
         $dbh->rollback if $oldAutoCommit;
         return "Could not load cust_main for $table ".$record->get($record->primary_key);
       }
+      # no clear record of name/address/etc used for transaction,
+      # but will load name/phone/id from customer if run as an object method,
+      # so we try that if we can
       my %tokenopts = (
         'payment_gateway' => $gateway,
         'method'          => 'CC',
         'payinfo'         => $record->payinfo,
         'paydate'         => $record->paydate,
       );
-      my $error = $cust_main->realtime_tokenize(\%tokenopts);
-      if ($cust_main->tokenized($tokenopts{'payinfo'})) { # implies no error
+      my $error = $cust_main
+                ? $cust_main->realtime_tokenize(\%tokenopts)
+                : FS::cust_main::Billing_Realtime->realtime_tokenize(\%tokenopts);
+      if (FS::cust_main::Billing_Realtime->tokenized($tokenopts{'payinfo'})) { # implies no error
         $record->payinfo($tokenopts{'payinfo'});
         $error = $record->replace;
       } else {
-        $error = 'Unknown error';
+        $error ||= 'Unknown error';
       }
       if ($error) {
         $search->DESTROY;

commit 6f2add8c2496952f0953ae066cfde3570610c98e
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Fri Nov 18 05:14:22 2016 -0600

    71513: Card tokenization [token_check]

diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm
index a2b1653..ec317ba 100644
--- a/FS/FS/Conf.pm
+++ b/FS/FS/Conf.pm
@@ -793,13 +793,6 @@ my $validate_email = sub { $_[0] =~
   },
   
   {
-    'key'         => 'no_saved_cardnumbers',
-    'section'     => 'credit_cards',
-    'description' => 'Do not allow credit card numbers to be written to the database.  Prevents realtime processing unless payment gateway supports tokenization.',
-    'type'        => 'checkbox',
-  },
-
-  {
     'key'         => 'credit-card-surcharge-percentage',
     'section'     => 'credit_cards',
     'description' => 'Add a credit card surcharge to invoices, as a % of the invoice total.  WARNING: Although recently permitted to US merchants in general, specific consumer protection laws may prohibit or restrict this practice in California, Colorado, Connecticut, Florda, Kansas, Maine, Massachusetts, New York, Oklahome, and Texas.  Surcharging is also generally prohibited in most countries outside the US, AU and UK.  When allowed, typically not permitted to be above 4%.',
diff --git a/FS/FS/Upgrade.pm b/FS/FS/Upgrade.pm
index 5a1ac2b..9c0a230 100644
--- a/FS/FS/Upgrade.pm
+++ b/FS/FS/Upgrade.pm
@@ -47,6 +47,10 @@ sub upgrade_config {
 
   my $conf = new FS::Conf;
 
+  # to simplify tokenization upgrades
+  die "Conf selfservice-payment_gateway no longer supported"
+    if conf->config('selfservice-payment_gateway');
+
   $conf->touch('payment_receipt')
     if $conf->exists('payment_receipt_email')
     || $conf->config('payment_receipt_msgnum');
@@ -392,6 +396,10 @@ sub upgrade_data {
     #duplicate history records
     'h_cust_svc'  => [],
 
+    # need before transaction tables, 
+    # blocks tokenization upgrade if deprecated features still in use
+    'agent_payment_gateway' => [],
+
     #populate cust_pay.otaker
     'cust_pay'    => [],
 
diff --git a/FS/FS/agent.pm b/FS/FS/agent.pm
index c102e7b..8aa78c2 100644
--- a/FS/FS/agent.pm
+++ b/FS/FS/agent.pm
@@ -238,31 +238,38 @@ sub ticketing_queue {
 
 Returns a payment gateway object (see L<FS::payment_gateway>) for this agent.
 
-Currently available options are I<nofatal>, I<invnum>, I<method>, 
-I<payinfo>, and I<thirdparty>.
+Currently available options are I<nofatal>, I<method>, I<thirdparty>,
+<conf> and I<load_gatewaynum>.
 
 If I<nofatal> is set, and no gateway is available, then the empty string
 will be returned instead of throwing a fatal exception.
 
-If I<invnum> is set to the number of an invoice (see L<FS::cust_bill>) then
-an attempt will be made to select a gateway suited for the taxes paid on 
-the invoice.
+The I<method> option can be used to influence the choice
+as well.  Presently only CHEK/ECHECK and PAYPAL methods are meaningful.
 
-The I<method> and I<payinfo> options can be used to influence the choice
-as well.  Presently only 'CC', 'ECHECK', and 'PAYPAL' methods are meaningful.
+If I<method> is CHEK/ECHECK and the default gateway is being returned,
+the business-onlinepayment-ach gateway will be returned if available.
 
-When the I<method> is 'CC' then the card number in I<payinfo> can direct
-this routine to route to a gateway suited for that type of card.
+If I<thirdparty> is set and the I<method> is PAYPAL, the defined paypal
+gateway will be returned.
 
-If I<thirdparty> is set, the defined self-service payment gateway will 
-be returned.
+If I<load_gatewaynum> exists, then either the specified gateway or the
+default gateway will be returned.  Agent overrides are ignored, and this can
+safely be called as a class method if this option is specified.  Not
+compatible with I<thirdparty>.
+
+Exsisting I<$conf> may be passed for efficiency.
 
 =cut
 
+# opts invnum/payinfo for cardtype/taxclass overrides no longer supported
+# any future overrides added here need to be reconciled with the tokenization process
+
 sub payment_gateway {
   my ( $self, %options ) = @_;
   
-  my $conf = new FS::Conf;
+  my $conf = $options{'conf'};
+  $conf ||= new FS::Conf;
 
   if ( $options{thirdparty} ) {
 
@@ -292,52 +299,17 @@ sub payment_gateway {
     }
   }
 
-  my $taxclass = '';
-  if ( $options{invnum} ) {
-
-    my $cust_bill = qsearchs('cust_bill', { 'invnum' => $options{invnum} } );
-    die "invnum ". $options{'invnum'}. " not found" unless $cust_bill;
-
-    my @part_pkg =
-      map  { $_->part_pkg }
-      grep { $_ }
-      map  { $_->cust_pkg }
-      $cust_bill->cust_bill_pkg;
-
-    my @taxclasses = map $_->taxclass, @part_pkg;
-
-    $taxclass = $taxclasses[0]
-      unless grep { $taxclasses[0] ne $_ } @taxclasses; #unless there are
-                                                        #different taxclasses
+  my ($override, $payment_gateway);
+  if (exists $options{'load_gatewaynum'}) { # no agent overrides if this opt is in use
+    if ($options{'load_gatewaynum'}) {
+      $payment_gateway = qsearchs('payment_gateway', { gatewaynumnum => $options{'load_gatewaynum'} } );
+      # always fatal
+      die "Could not load payment gateway ".$options{'load_gatewaynum'} unless $payment_gateway;
+    } # else use default, loaded below
+  } else {
+    $override = qsearchs('agent_payment_gateway', { agentnum => $self->agentnum } );
   }
 
-  #look for an agent gateway override first
-  my $cardtype = '';
-  if ( $options{method} ) {
-    if ( $options{method} eq 'CC' && $options{payinfo} ) {
-      $cardtype = cardtype($options{payinfo});
-    } elsif ( $options{method} eq 'ECHECK' ) {
-      $cardtype = 'ACH';
-    } else {
-      $cardtype = $options{method}
-    }
-  }
-
-  my $override =
-       qsearchs('agent_payment_gateway', { agentnum => $self->agentnum,
-                                           cardtype => $cardtype,
-                                           taxclass => $taxclass,       } )
-    || qsearchs('agent_payment_gateway', { agentnum => $self->agentnum,
-                                           cardtype => '',
-                                           taxclass => $taxclass,       } )
-    || qsearchs('agent_payment_gateway', { agentnum => $self->agentnum,
-                                           cardtype => $cardtype,
-                                           taxclass => '',              } )
-    || qsearchs('agent_payment_gateway', { agentnum => $self->agentnum,
-                                           cardtype => '',
-                                           taxclass => '',              } );
-
-  my $payment_gateway;
   if ( $override ) { #use a payment gateway override
 
     $payment_gateway = $override->payment_gateway;
@@ -345,11 +317,13 @@ sub payment_gateway {
     $payment_gateway->gateway_namespace('Business::OnlinePayment')
       unless $payment_gateway->gateway_namespace;
 
-  } else { #use the standard settings from the config
+  } elsif (!$payment_gateway) { #use the standard settings from the config
 
     # the standard settings from the config could be moved to a null agent
     # agent_payment_gateway referenced payment_gateway
 
+    # remember, this block might be run as a class method if false load_gatewaynum exists
+
     unless ( $conf->exists('business-onlinepayment') ) {
       if ( $options{'nofatal'} ) {
         return '';
diff --git a/FS/FS/agent_payment_gateway.pm b/FS/FS/agent_payment_gateway.pm
index e71ed21..4991c19 100644
--- a/FS/FS/agent_payment_gateway.pm
+++ b/FS/FS/agent_payment_gateway.pm
@@ -111,6 +111,21 @@ sub check {
   $self->SUPER::check;
 }
 
+sub _upgrade_data {
+  # to simplify tokenization upgrades
+  die "Agent taxclass override no longer supported"
+    if qsearch({
+      'table' => 'agent_payment_gateway',
+      'extra_sql' => ' WHERE taxclass IS NOT NULL AND taxclass != \'\'',
+    });
+  die "Agent cardtype override no longer supported"
+    if qsearch({
+      'table' => 'agent_payment_gateway',
+      'extra_sql' => ' WHERE cardtype IS NOT NULL AND cardtype != \'\'',
+    });
+  return '';
+}
+
 =item payment_gateway
 
 =back
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 34966ce..48b6ee6 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -248,8 +248,9 @@ sub _bop_auth {
   );
 }
 
+### not a method!
 sub _bop_options {
-  my ($self, $options) = @_;
+  my ($options) = @_;
 
   $options->{payment_gateway}->gatewaynum
     ? $options->{payment_gateway}->options
@@ -692,7 +693,7 @@ sub realtime_bop {
     split( /\s*\,\s*/, $payment_gateway->gateway_action );
 
   my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
+                                    _bop_options(\%options),
                                   );
 
   $transaction->content(
@@ -752,7 +753,7 @@ sub realtime_bop {
 
     my $capture =
       new Business::OnlinePayment( $payment_gateway->gateway_module,
-                                   $self->_bop_options(\%options),
+                                   _bop_options(\%options),
                                  );
 
     my %capture = (
@@ -1283,7 +1284,7 @@ sub realtime_botpp_capture {
 
   my $transaction =
     new Business::OnlineThirdPartyPayment( $payment_gateway->gateway_module,
-                                           $self->_bop_options(\%options),
+                                           _bop_options(\%options),
                                          );
 
   $transaction->reference({ %options }); 
@@ -1905,7 +1906,7 @@ sub realtime_verify_bop {
     warn Dumper($cust_pay_pending) if $DEBUG > 2;
 
     $transaction = new $namespace( $payment_gateway->gateway_module,
-                                   $self->_bop_options(\%options),
+                                   _bop_options(\%options),
                                     );
 
     $transaction->content(
@@ -1953,7 +1954,7 @@ sub realtime_verify_bop {
                      : '';
 
       my $reverse = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
+                                    _bop_options(\%options),
                                   );
 
       $reverse->content( 'action'        => 'Reverse Authorization',
@@ -2217,7 +2218,7 @@ sub realtime_tokenize {
   ###
 
   my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
+                                    _bop_options(\%options),
                                   );
 
   return '' unless $transaction->can('info');
@@ -2323,46 +2324,61 @@ sub tokenized {
   FS::cust_pay->tokenized($payinfo);
 }
 
-=item remove_card_numbers
+=item token_check
 
-NOT AN OBJECT METHOD.  Acts on all customers.  Placed here because it makes
+NOT A METHOD.  Acts on all customers.  Placed here because it makes
 use of module-internal methods, and to keep everything that uses
 Billing::OnlinePayment all in one place.
 
-Removes all stored card numbers from payinfo in cust_payby and 
+Tokenizes all tokenizable card numbers from payinfo in cust_payby and 
 CARD transactions in cust_pay_pending, cust_pay, cust_pay_void and cust_refund.
-Will fail if cust_payby records can't be tokenized.  Transaction records that
-cannot be tokenized will have their payinfo replaced with their paymask.
-
-THIS WILL OVERWRITE STORED PAYINFO ON OLD TRANSACTIONS.
-
-If the gateway originally used for the transaction can't tokenize, this may
-prevent the transaction from being voided or refunded.  Hence, it should
-not (yet) be run as part of a regular upgrade.  This is only intended to be
-run on systems with current gateways that tokenize, after the window has
-passed for voiding/refunding transactions from previous gateways, in order 
-to remove all real card numbers from the system.
 
-Also sets the no_saved_cardnumbers conf, to keep things this way.
+If all configured gateways have the ability to tokenize, then detection of
+an untokenizable record will cause a fatal error.
 
 =cut
 
-# ??? probably should add MCRD handling to this
-
-sub remove_card_numbers {
-  # no input, always does the same thing
-
-  my $cache = {}; #cache for module info
+sub token_check {
+  # no input, acts on all customers
 
   eval "use FS::Cursor";  
   return "Error initializing FS::Cursor: ".$@ if $@;
 
-  my $oldAutoCommit = $FS::UID::AutoCommit;
-  local $FS::UID::AutoCommit = 0;
   my $dbh = dbh;
 
-  # turn this on
-  $conf->touch('no_saved_cardnumbers');
+  # get list of all gateways in table (not counting default gateway)
+  my $cache = {}; #cache for module info
+  my $sth = $dbh->prepare('SELECT DISTINCT gatewaynum FROM payment_gateway')
+    or die $dbh->errstr;
+  $sth->execute or die $sth->errstr;
+  my @gatewaynums;
+  while (my $row = $sth->fetchrow_hashref) {
+    push(@gatewaynums,$row->{'gatewaynum'});
+  }
+  $sth->finish;
+
+  # look for a gateway that can't tokenize
+  my $disallow_untokenized = 1;
+  foreach my $gatewaynum ('', at gatewaynums) {
+    my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum, nofatal => 1 );
+    if (!$gateway) { # already died if $gatewaynum
+      # no default gateway, no promise to tokenize
+      # can just load other gateways as-needeed below
+      $disallow_untokenized = 0;
+      last;
+    }
+    my $info = _token_check_gateway_info($cache,$gateway);
+    return $info unless ref($info); # means it's an error message
+    unless ($info->{'can_tokenize'}) {
+      # a configured gateway can't tokenize, that's all we need to know right now
+      # can just load other gateways as-needeed below
+      $disallow_untokenized = 0;
+      last;
+    }
+  }
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
 
   ### Tokenize cust_payby
 
@@ -2372,24 +2388,19 @@ sub remove_card_numbers {
       next if $cust_payby->tokenized;
       # load gateway first, just so we can cache it
       my $payment_gateway = $cust_main->_payment_gateway({
-        'payinfo' => $cust_payby->payinfo, # for cardtype agent overrides
         'nofatal' => 1, # handle error smoothly below
-        # invnum -- XXX need to figure out how to handle taxclass overrides
       });
       unless ($payment_gateway) {
+        # no reason to have untokenized card numbers saved if no gateway,
+        #   but only fatal if we expected everyone to tokenize card numbers
+        next unless $disallow_untokenized;
         $cust_search->DESTROY;
         $dbh->rollback if $oldAutoCommit;
         return "No gateway found for custnum ".$cust_main->custnum;
       }
-      my $info = $cust_main->_remove_card_numbers_gateway_info($cache,$payment_gateway);
-      unless (ref($info) && $info->{'can_tokenize'}) {
-        $cust_search->DESTROY;
-        $dbh->rollback if $oldAutoCommit;
-        my $error = ref($info)
-          ? "Gateway ".$payment_gateway->gatewaynum." cannot tokenize, for custnum ".$cust_main->custnum
-          : $info;
-        return $error;
-      }
+      my $info = _token_check_gateway_info($cache,$payment_gateway);
+      # no fail here--a configured gateway can't tokenize, so be it
+      next unless ref($info) && $info->{'can_tokenize'};
       my %tokenopts = (
         'payment_gateway' => $payment_gateway,
         'cust_payby'      => $cust_payby,
@@ -2398,7 +2409,7 @@ sub remove_card_numbers {
       if ($cust_payby->tokenized) { # implies no error
         $error = $cust_payby->replace;
       } else {
-        $error = 'Unknown error';
+        $error ||= 'Unknown error';
       }
       if ($error) {
         $cust_search->DESTROY;
@@ -2419,66 +2430,77 @@ sub remove_card_numbers {
     },$dbh);
     while (my $record = $search->fetch) {
       next if $record->tokenized;
-      next if !$record->payinfo; #shouldn't happen, but just in case, no need to mask
-      next if $record->payinfo =~ /N\/A/; # ??? Not sure what's up with these, but no need to mask
-      next if $record->payinfo eq $record->paymask; #already masked
-      my $old_gateway;
-      if (my $old_gatewaynum = $record->gatewaynum) {
-        $old_gateway = 
-          qsearchs('payment_gateway',{ 'gatewaynum' => $old_gatewaynum, });
-        # not erring out if gateway can't be found, just use paymask
+      next if !$record->payinfo; #shouldn't happen, but at least it's not a card number
+      next if $record->payinfo =~ /N\/A/; # ??? Not sure why we do this, but it's not a card number
+
+      # don't use customer agent gateway here, use the gatewaynum specified by the record
+      my $gatewaynum = $record->gatewaynum || '';
+      my $gateway = FS::agent->payment_gateway( load_gatewaynum => $gatewaynum );
+      unless ($gateway) { # already died if $gatewaynum
+        # only fatal if we expected everyone to tokenize
+        next unless $disallow_untokenized;
+        $search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return "No gateway found for $table ".$record->get($record->primary_key);
       }
-      # first try to tokenize
+      my $info = _token_check_gateway_info($cache,$gateway);
+      unless (ref($info)) {
+        # only throws error if Business::OnlinePayment won't load,
+        #   which is just cause to abort this whole process
+        $search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return $info; # error message
+      }
+
+      # a configured gateway can't tokenize, move along
+      next unless $info->{'can_tokenize'};
+
       my $cust_main = $record->cust_main;
-      if ($cust_main && $old_gateway) {
-        my $info = $cust_main->_remove_card_numbers_gateway_info($cache,$old_gateway);
-        unless (ref($info)) {
-          # only throws error if Business::OnlinePayment won't load,
-          #   which is just cause to abort this whole process
-          $search->DESTROY;
-          $dbh->rollback if $oldAutoCommit;
-          return $info;
-        }
-        if ($info->{'can_tokenize'}) {
-          my %tokenopts = (
-            'payment_gateway' => $old_gateway,
-            'method'          => 'CC',
-            'payinfo'         => $record->payinfo,
-            'paydate'         => $record->paydate,
-          );
-          my $error = $cust_main->realtime_tokenize(\%tokenopts);
-          if ($cust_main->tokenized($tokenopts{'payinfo'})) { # implies no error
-            $record->payinfo($tokenopts{'payinfo'});
-            $error = $record->replace;
-          } else {
-            $error = 'Unknown error';
-          }
-          if ($error) {
-            $search->DESTROY;
-            $dbh->rollback if $oldAutoCommit;
-            return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
-          }
-          next;
-        }
+      unless ($cust_main) {
+        # might happen for cust_pay_pending for failed verify records,
+        #   in which case it *should* already be tokenized if possible
+        #   but only get strict about it if we're expecting full tokenization
+        next if 
+          $table eq 'cust_pay_pending'
+            && $record->{'custnum_pending'}
+            && !$disallow_untokenized;
+        # XXX we currently need a $cust_main to run realtime_tokenize
+        #     even if we made it a class method, wouldn't have access to payname/etc.
+        #     fail for now, but probably could handle this better...
+        # everything else should absolutely have a cust_main
+        $search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return "Could not load cust_main for $table ".$record->get($record->primary_key);
+      }
+      my %tokenopts = (
+        'payment_gateway' => $gateway,
+        'method'          => 'CC',
+        'payinfo'         => $record->payinfo,
+        'paydate'         => $record->paydate,
+      );
+      my $error = $cust_main->realtime_tokenize(\%tokenopts);
+      if ($cust_main->tokenized($tokenopts{'payinfo'})) { # implies no error
+        $record->payinfo($tokenopts{'payinfo'});
+        $error = $record->replace;
+      } else {
+        $error = 'Unknown error';
       }
-      # can't tokenize, so just replace with paymask
-      $record->set('payinfo',$record->paymask); #deliberately evade ->payinfo() remasking effects
-      my $error = $record->replace;
       if ($error) {
         $search->DESTROY;
         $dbh->rollback if $oldAutoCommit;
-        return "Error masking payinfo for $table ".$record->get($record->primary_key).": ".$error;
+        return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
       }
-    }
-  }
+    } # end record loop
+  } # end table loop
 
   $dbh->commit if $oldAutoCommit;
 
   return '';
 }
 
-sub _remove_card_numbers_gateway_info {
-  my ($self,$cache,$payment_gateway) = @_;
+# not a method!
+sub _token_check_gateway_info {
+  my ($cache,$payment_gateway) = @_;
 
   return $cache->{$payment_gateway->gateway_module}
     if $cache->{$payment_gateway->gateway_module};
@@ -2499,7 +2521,7 @@ sub _remove_card_numbers_gateway_info {
   }
 
   my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options({ 'payment_gateway' => $payment_gateway }),
+                                    _bop_options({ 'payment_gateway' => $payment_gateway }),
                                   );
 
   return $info unless $transaction->can('info');
@@ -2510,6 +2532,7 @@ sub _remove_card_numbers_gateway_info {
     if $supported_actions{'CC'}
       && grep /^Tokenize$/, @{$supported_actions{'CC'}};
 
+  # not using this any more, but for future reference...
   $info->{'void_requires_card'} = 1
     if $transaction->info('CC_void_requires_card');
 
diff --git a/FS/FS/cust_pay.pm b/FS/FS/cust_pay.pm
index e0a7143..b15920b 100644
--- a/FS/FS/cust_pay.pm
+++ b/FS/FS/cust_pay.pm
@@ -540,7 +540,8 @@ otherwise returns false.
 
 sub replace {
   my $self = shift;
-  return "Can't modify closed payment" if $self->closed =~ /^Y/i;
+  return "Can't modify closed payment"
+    if $self->closed =~ /^Y/i && !$FS::payinfo_Mixin::allow_closed_replace;
   $self->SUPER::replace(@_);
 }
 
diff --git a/FS/FS/cust_refund.pm b/FS/FS/cust_refund.pm
index 4d2baa5..12ab0d6 100644
--- a/FS/FS/cust_refund.pm
+++ b/FS/FS/cust_refund.pm
@@ -289,7 +289,8 @@ otherwise returns false.
 
 sub replace {
   my $self = shift;
-  return "Can't modify closed refund" if $self->closed =~ /^Y/i;
+  return "Can't modify closed refund" 
+    if $self->closed =~ /^Y/i && !$FS::payinfo_Mixin::allow_closed_replace;
   $self->SUPER::replace(@_);
 }
 
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 7a3dcf0..2f50312 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -8,7 +8,8 @@ use FS::UID qw(driver_name);
 use FS::Cursor;
 use Time::Local qw(timelocal);
 
-use vars qw($ignore_masked_payinfo);
+# allow_closed_replace only relevant to cust_pay/cust_refund, for upgrade tokenizing
+use vars qw( $ignore_masked_payinfo $allow_closed_replace );
 
 =head1 NAME
 
@@ -214,8 +215,6 @@ sub payinfo_check {
         $self->payinfo($1);
         validate($self->payinfo) or return "Illegal credit card number";
         return "Unknown card type" if $cardtype eq "Unknown";
-        return "Card number not tokenized"
-          if $conf->exists('no_saved_cardnumbers') && !$self->tokenized;
       } else {
         $self->payinfo('N/A'); #??? re-masks card
       }
diff --git a/FS/FS/payinfo_transaction_Mixin.pm b/FS/FS/payinfo_transaction_Mixin.pm
index 6e4b511..c27d049 100644
--- a/FS/FS/payinfo_transaction_Mixin.pm
+++ b/FS/FS/payinfo_transaction_Mixin.pm
@@ -104,10 +104,6 @@ sub payinfo_check {
 
   my $conf = new FS::Conf;
 
-  # allow masked payinfo if we never save card numbers
-  local $FS::payinfo_Mixin::ignore_masked_payinfo = 
-    $conf->exists('no_saved_cardnumbers') ? 1 : $FS::payinfo_Mixin::ignore_masked_payinfo;
-
   $self->SUPER::payinfo_check()
   || $self->ut_numbern('gatewaynum')
   # not ut_foreign_keyn, it causes upgrades to fail

commit 868e9dd529dc43fd523a6883ee72ec9b22e11b90
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 15 03:08:29 2016 -0600

    71513: Card tokenization [remove_card_numbers subroutine]

diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm
index ec317ba..a2b1653 100644
--- a/FS/FS/Conf.pm
+++ b/FS/FS/Conf.pm
@@ -793,6 +793,13 @@ my $validate_email = sub { $_[0] =~
   },
   
   {
+    'key'         => 'no_saved_cardnumbers',
+    'section'     => 'credit_cards',
+    'description' => 'Do not allow credit card numbers to be written to the database.  Prevents realtime processing unless payment gateway supports tokenization.',
+    'type'        => 'checkbox',
+  },
+
+  {
     'key'         => 'credit-card-surcharge-percentage',
     'section'     => 'credit_cards',
     'description' => 'Add a credit card surcharge to invoices, as a % of the invoice total.  WARNING: Although recently permitted to US merchants in general, specific consumer protection laws may prohibit or restrict this practice in California, Colorado, Connecticut, Florda, Kansas, Maine, Massachusetts, New York, Oklahome, and Texas.  Surcharging is also generally prohibited in most countries outside the US, AU and UK.  When allowed, typically not permitted to be above 4%.',
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index fcc573e..34966ce 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -413,15 +413,17 @@ sub realtime_bop {
 
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
-  if (!$self->tokenized($options{'payinfo'})) {
+  if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
     my $token_error = $self->realtime_tokenize(\%options);
     return $token_error if $token_error;
     # in theory, all cust_payby will be tokenized during original save,
     # so we shouldn't get here with opt cust_payby...but just in case...
-    if ($options{'cust_payby'}) {
+    if ($options{'cust_payby'} && $self->tokenized($options{'payinfo'})) {
       $token_error = $options{'cust_payby'}->replace;
       return $token_error if $token_error;
     }
+    return "Cannot tokenize card info"
+      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ### 
@@ -1765,11 +1767,13 @@ sub realtime_verify_bop {
 
   # possibly run a separate transaction to tokenize card number,
   #   so that we never store tokenized card info in cust_pay_pending
-  if (!$self->tokenized($options{'payinfo'})) {
+  if (($options{method} eq 'CC') && !$self->tokenized($options{'payinfo'})) {
     my $token_error = $self->realtime_tokenize(\%options);
     return $token_error if $token_error;
     #important that we not replace cust_payby here,
     #because cust_payby->replace uses realtime_verify_bop!
+    return "Cannot tokenize card info"
+      if $conf->exists('no_saved_cardnumbers') && !$self->tokenized($options{'payinfo'});
   }
 
   ###
@@ -2216,6 +2220,8 @@ sub realtime_tokenize {
                                     $self->_bop_options(\%options),
                                   );
 
+  return '' unless $transaction->can('info');
+
   my %supported_actions = $transaction->info('supported_actions');
   return '' unless $supported_actions{'CC'} and grep(/^Tokenize$/,@{$supported_actions{'CC'}});
 
@@ -2317,12 +2323,205 @@ sub tokenized {
   FS::cust_pay->tokenized($payinfo);
 }
 
+=item remove_card_numbers
+
+NOT AN OBJECT METHOD.  Acts on all customers.  Placed here because it makes
+use of module-internal methods, and to keep everything that uses
+Billing::OnlinePayment all in one place.
+
+Removes all stored card numbers from payinfo in cust_payby and 
+CARD transactions in cust_pay_pending, cust_pay, cust_pay_void and cust_refund.
+Will fail if cust_payby records can't be tokenized.  Transaction records that
+cannot be tokenized will have their payinfo replaced with their paymask.
+
+THIS WILL OVERWRITE STORED PAYINFO ON OLD TRANSACTIONS.
+
+If the gateway originally used for the transaction can't tokenize, this may
+prevent the transaction from being voided or refunded.  Hence, it should
+not (yet) be run as part of a regular upgrade.  This is only intended to be
+run on systems with current gateways that tokenize, after the window has
+passed for voiding/refunding transactions from previous gateways, in order 
+to remove all real card numbers from the system.
+
+Also sets the no_saved_cardnumbers conf, to keep things this way.
+
+=cut
+
+# ??? probably should add MCRD handling to this
+
+sub remove_card_numbers {
+  # no input, always does the same thing
+
+  my $cache = {}; #cache for module info
+
+  eval "use FS::Cursor";  
+  return "Error initializing FS::Cursor: ".$@ if $@;
+
+  my $oldAutoCommit = $FS::UID::AutoCommit;
+  local $FS::UID::AutoCommit = 0;
+  my $dbh = dbh;
+
+  # turn this on
+  $conf->touch('no_saved_cardnumbers');
+
+  ### Tokenize cust_payby
+
+  my $cust_search = FS::Cursor->new({ table => 'cust_main' },$dbh);
+  while (my $cust_main = $cust_search->fetch) {
+    foreach my $cust_payby ($cust_main->cust_payby('CARD','DCRD')) {
+      next if $cust_payby->tokenized;
+      # load gateway first, just so we can cache it
+      my $payment_gateway = $cust_main->_payment_gateway({
+        'payinfo' => $cust_payby->payinfo, # for cardtype agent overrides
+        'nofatal' => 1, # handle error smoothly below
+        # invnum -- XXX need to figure out how to handle taxclass overrides
+      });
+      unless ($payment_gateway) {
+        $cust_search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return "No gateway found for custnum ".$cust_main->custnum;
+      }
+      my $info = $cust_main->_remove_card_numbers_gateway_info($cache,$payment_gateway);
+      unless (ref($info) && $info->{'can_tokenize'}) {
+        $cust_search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        my $error = ref($info)
+          ? "Gateway ".$payment_gateway->gatewaynum." cannot tokenize, for custnum ".$cust_main->custnum
+          : $info;
+        return $error;
+      }
+      my %tokenopts = (
+        'payment_gateway' => $payment_gateway,
+        'cust_payby'      => $cust_payby,
+      );
+      my $error = $cust_main->realtime_tokenize(\%tokenopts);
+      if ($cust_payby->tokenized) { # implies no error
+        $error = $cust_payby->replace;
+      } else {
+        $error = 'Unknown error';
+      }
+      if ($error) {
+        $cust_search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return "Error tokenizing cust_payby ".$cust_payby->custpaybynum.": ".$error;
+      }
+    }
+  }
+
+  ### Tokenize/mask transaction tables
+
+  # grep assistance:
+  #   $cust_pay_pending->replace, $cust_pay->replace, $cust_pay_void->replace, $cust_refund->replace all run here
+  foreach my $table ( qw(cust_pay_pending cust_pay cust_pay_void cust_refund) ) {
+    my $search = FS::Cursor->new({
+      table     => $table,
+      hashref   => { 'payby' => 'CARD' },
+    },$dbh);
+    while (my $record = $search->fetch) {
+      next if $record->tokenized;
+      next if !$record->payinfo; #shouldn't happen, but just in case, no need to mask
+      next if $record->payinfo =~ /N\/A/; # ??? Not sure what's up with these, but no need to mask
+      next if $record->payinfo eq $record->paymask; #already masked
+      my $old_gateway;
+      if (my $old_gatewaynum = $record->gatewaynum) {
+        $old_gateway = 
+          qsearchs('payment_gateway',{ 'gatewaynum' => $old_gatewaynum, });
+        # not erring out if gateway can't be found, just use paymask
+      }
+      # first try to tokenize
+      my $cust_main = $record->cust_main;
+      if ($cust_main && $old_gateway) {
+        my $info = $cust_main->_remove_card_numbers_gateway_info($cache,$old_gateway);
+        unless (ref($info)) {
+          # only throws error if Business::OnlinePayment won't load,
+          #   which is just cause to abort this whole process
+          $search->DESTROY;
+          $dbh->rollback if $oldAutoCommit;
+          return $info;
+        }
+        if ($info->{'can_tokenize'}) {
+          my %tokenopts = (
+            'payment_gateway' => $old_gateway,
+            'method'          => 'CC',
+            'payinfo'         => $record->payinfo,
+            'paydate'         => $record->paydate,
+          );
+          my $error = $cust_main->realtime_tokenize(\%tokenopts);
+          if ($cust_main->tokenized($tokenopts{'payinfo'})) { # implies no error
+            $record->payinfo($tokenopts{'payinfo'});
+            $error = $record->replace;
+          } else {
+            $error = 'Unknown error';
+          }
+          if ($error) {
+            $search->DESTROY;
+            $dbh->rollback if $oldAutoCommit;
+            return "Error tokenizing $table ".$record->get($record->primary_key).": ".$error;
+          }
+          next;
+        }
+      }
+      # can't tokenize, so just replace with paymask
+      $record->set('payinfo',$record->paymask); #deliberately evade ->payinfo() remasking effects
+      my $error = $record->replace;
+      if ($error) {
+        $search->DESTROY;
+        $dbh->rollback if $oldAutoCommit;
+        return "Error masking payinfo for $table ".$record->get($record->primary_key).": ".$error;
+      }
+    }
+  }
+
+  $dbh->commit if $oldAutoCommit;
+
+  return '';
+}
+
+sub _remove_card_numbers_gateway_info {
+  my ($self,$cache,$payment_gateway) = @_;
+
+  return $cache->{$payment_gateway->gateway_module}
+    if $cache->{$payment_gateway->gateway_module};
+
+  my $info = {};
+  $cache->{$payment_gateway->gateway_module} = $info;
+
+  my $namespace = $payment_gateway->gateway_namespace;
+  return $info unless $namespace eq 'Business::OnlinePayment';
+  $info->{'is_bop'} = 1;
+
+  # only need to load this once,
+  # don't want to load if nothing is_bop
+  unless ($cache->{'Business::OnlinePayment'}) {
+    eval "use $namespace";  
+    return "Error initializing Business:OnlinePayment: ".$@ if $@;
+    $cache->{'Business::OnlinePayment'} = 1;
+  }
+
+  my $transaction = new $namespace( $payment_gateway->gateway_module,
+                                    $self->_bop_options({ 'payment_gateway' => $payment_gateway }),
+                                  );
+
+  return $info unless $transaction->can('info');
+  $info->{'can_info'} = 1;
+
+  my %supported_actions = $transaction->info('supported_actions');
+  $info->{'can_tokenize'} = 1
+    if $supported_actions{'CC'}
+      && grep /^Tokenize$/, @{$supported_actions{'CC'}};
+
+  $info->{'void_requires_card'} = 1
+    if $transaction->info('CC_void_requires_card');
+
+  $cache->{$payment_gateway->gateway_module} = $info;
+
+  return $info;
+}
+
 =back
 
 =head1 BUGS
 
-Not autoloaded.
-
 =head1 SEE ALSO
 
 L<FS::cust_main>, L<FS::cust_main::Billing>
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index dfcce2f..7a3dcf0 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -194,6 +194,8 @@ sub payinfo_check {
   FS::payby->can_payby($self->table, $self->payby)
     or return "Illegal payby: ". $self->payby;
 
+  my $conf = new FS::Conf;
+
   if ( $self->payby eq 'CARD' && ! $self->is_encrypted($self->payinfo) ) {
 
     my $payinfo = $self->payinfo;
@@ -212,8 +214,10 @@ sub payinfo_check {
         $self->payinfo($1);
         validate($self->payinfo) or return "Illegal credit card number";
         return "Unknown card type" if $cardtype eq "Unknown";
+        return "Card number not tokenized"
+          if $conf->exists('no_saved_cardnumbers') && !$self->tokenized;
       } else {
-        $self->payinfo('N/A'); #???
+        $self->payinfo('N/A'); #??? re-masks card
       }
     }
   } else {
diff --git a/FS/FS/payinfo_transaction_Mixin.pm b/FS/FS/payinfo_transaction_Mixin.pm
index 50659ac..6e4b511 100644
--- a/FS/FS/payinfo_transaction_Mixin.pm
+++ b/FS/FS/payinfo_transaction_Mixin.pm
@@ -102,7 +102,11 @@ auth, and order_number) as well as payby and payinfo
 sub payinfo_check {
   my $self = shift;
 
-  # All of these can be null, so in principle this could go in payinfo_Mixin.
+  my $conf = new FS::Conf;
+
+  # allow masked payinfo if we never save card numbers
+  local $FS::payinfo_Mixin::ignore_masked_payinfo = 
+    $conf->exists('no_saved_cardnumbers') ? 1 : $FS::payinfo_Mixin::ignore_masked_payinfo;
 
   $self->SUPER::payinfo_check()
   || $self->ut_numbern('gatewaynum')

commit 995b2edc47e2285db5c2cd64ef0783f2cbc37ee8
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 15 02:49:35 2016 -0600

    71513: Card tokenization [bug fix to selfservice-payment_gateway removal]

diff --git a/FS/FS/ClientAPI/Signup.pm b/FS/FS/ClientAPI/Signup.pm
index 7fad7b3..5ced42b 100644
--- a/FS/FS/ClientAPI/Signup.pm
+++ b/FS/FS/ClientAPI/Signup.pm
@@ -1104,7 +1104,7 @@ sub capture_payment {
   my $conf = new FS::Conf;
 
   my $url = $packet->{url};
-  my $payment_gateway = $payment_gateway = qsearchs('payment_gateway', 
+  my $payment_gateway = qsearchs('payment_gateway', 
         { 'gateway_callback_url' => popurl(0, $url) } 
       );
   if (!$payment_gateway) { 

commit 5f2c60fc31d06443fb43b30f62a4d2789aad55cd
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Fri Nov 11 21:02:01 2016 -0600

    71513: Card tokenization [removed selfservice-payment_gateway]

diff --git a/FS/FS/ClientAPI/MyAccount.pm b/FS/FS/ClientAPI/MyAccount.pm
index 091d6ac..4a878f8 100644
--- a/FS/FS/ClientAPI/MyAccount.pm
+++ b/FS/FS/ClientAPI/MyAccount.pm
@@ -401,20 +401,12 @@ sub payment_gateway {
   my $conf = new FS::Conf;
   my $cust_main = shift;
   my $cust_payby = shift;
-  my $gatewaynum = $conf->config('selfservice-payment_gateway');
-  if ( $gatewaynum ) {
-    my $pg = qsearchs('payment_gateway', { gatewaynum => $gatewaynum });
-    die "configured gatewaynum $gatewaynum not found!" if !$pg;
-    return $pg;
-  }
-  else {
-    return '' if ! FS::payby->realtime($cust_payby);
-    my $pg = $cust_main->agent->payment_gateway(
-      'method'  => FS::payby->payby2bop($cust_payby),
-      'nofatal' => 1
-    );
-    return $pg;
-  }
+  return '' if ! FS::payby->realtime($cust_payby);
+  my $pg = $cust_main->agent->payment_gateway(
+    'method'  => FS::payby->payby2bop($cust_payby),
+    'nofatal' => 1
+  );
+  return $pg;
 }
 
 sub access_info {
diff --git a/FS/FS/ClientAPI/Signup.pm b/FS/FS/ClientAPI/Signup.pm
index e11a47a..7fad7b3 100644
--- a/FS/FS/ClientAPI/Signup.pm
+++ b/FS/FS/ClientAPI/Signup.pm
@@ -344,20 +344,11 @@ sub signup_info {
     my @paybys = @{ $signup_info->{'payby'} };
     $signup_info->{'hide_payment_fields'} = [];
 
-    my $gatewaynum = $conf->config('selfservice-payment_gateway');
-    my $force_gateway;
-    if ( $gatewaynum ) {
-      $force_gateway = qsearchs('payment_gateway', { gatewaynum => $gatewaynum });
-      warn "using forced gateway #$gatewaynum - " .
-        $force_gateway->gateway_username . '@' . $force_gateway->gateway_module
-        if $DEBUG > 1;
-      die "configured gatewaynum $gatewaynum not found!" if !$force_gateway;
-    }
     foreach my $payby (@paybys) {
       warn "$me checking $payby payment fields\n" if $DEBUG > 1;
       my $hide = 0;
       if ( FS::payby->realtime($payby) ) {
-        my $gateway = $force_gateway || 
+        my $gateway = 
           $agent->payment_gateway( 'method'  => FS::payby->payby2bop($payby),
                                    'nofatal' => 1,
                                  );
@@ -627,17 +618,9 @@ sub new_customer {
     return { 'error' => "Unknown reseller" }
       unless $agent;
 
-    my $gw;
-    my $gatewaynum = $conf->config('selfservice-payment_gateway');
-    if ( $gatewaynum ) {
-      $gw = qsearchs('payment_gateway', { gatewaynum => $gatewaynum });
-      die "configured gatewaynum $gatewaynum not found!" if !$gw;
-    }
-    else {
-      $gw = $agent->payment_gateway( 'method'  => FS::payby->payby2bop($payby),
-                                     'nofatal' => 1,
+    my $gw = $agent->payment_gateway( 'method'  => FS::payby->payby2bop($payby),
+                                      'nofatal' => 1,
                                     );
-    }
 
     $cust_main->payby('BILL')   # MCRD better?  no, that's for something else
       if $gw && $gw->gateway_namespace eq 'Business::OnlineThirdPartyPayment';
@@ -1120,36 +1103,28 @@ sub capture_payment {
 
   my $conf = new FS::Conf;
 
-  my $payment_gateway;
-  if ( my $gwnum = $conf->config('selfservice-payment_gateway') ) {
-    $payment_gateway = qsearchs('payment_gateway', { 'gatewaynum' => $gwnum })
-      or die "configured gatewaynum $gwnum not found!";
-  }
-  else {
-    my $url = $packet->{url};
-
-    $payment_gateway = qsearchs('payment_gateway', 
+  my $url = $packet->{url};
+  my $payment_gateway = $payment_gateway = qsearchs('payment_gateway', 
         { 'gateway_callback_url' => popurl(0, $url) } 
       );
-    if (!$payment_gateway) { 
-
-      my ( $processor, $login, $password, $action, @bop_options ) =
-        $conf->config('business-onlinepayment');
-      $action ||= 'normal authorization';
-      pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
-      die "No real-time processor is enabled - ".
-          "did you set the business-onlinepayment configuration value?\n"
-        unless $processor;
-
-      $payment_gateway = new FS::payment_gateway( {
-        gateway_namespace => $conf->config('business-onlinepayment-namespace'),
-        gateway_module    => $processor,
-        gateway_username  => $login,
-        gateway_password  => $password,
-        gateway_action    => $action,
-        options   => [ ( @bop_options ) ],
-      });
-    }
+  if (!$payment_gateway) { 
+
+    my ( $processor, $login, $password, $action, @bop_options ) =
+      $conf->config('business-onlinepayment');
+    $action ||= 'normal authorization';
+    pop @bop_options if scalar(@bop_options) % 2 && $bop_options[-1] =~ /^\s*$/;
+    die "No real-time processor is enabled - ".
+        "did you set the business-onlinepayment configuration value?\n"
+      unless $processor;
+
+    $payment_gateway = new FS::payment_gateway( {
+      gateway_namespace => $conf->config('business-onlinepayment-namespace'),
+      gateway_module    => $processor,
+      gateway_username  => $login,
+      gateway_password  => $password,
+      gateway_action    => $action,
+      options   => [ ( @bop_options ) ],
+    });
   }
  
   die "No real-time third party processor is enabled - ".
diff --git a/FS/FS/Conf.pm b/FS/FS/Conf.pm
index 4c87f93..ec317ba 100644
--- a/FS/FS/Conf.pm
+++ b/FS/FS/Conf.pm
@@ -2208,8 +2208,8 @@ and customer address. Include units.',
 
   {
     'key'         => 'selfservice-payment_gateway',
-    'section'     => 'self-service',
-    'description' => 'Force the use of this payment gateway for self-service.',
+    'section'     => 'deprecated',
+    'description' => '(no longer supported) Force the use of this payment gateway for self-service.',
     %payment_gateway_options,
   },
 
diff --git a/FS/FS/agent.pm b/FS/FS/agent.pm
index fc23433..c102e7b 100644
--- a/FS/FS/agent.pm
+++ b/FS/FS/agent.pm
@@ -265,24 +265,14 @@ sub payment_gateway {
   my $conf = new FS::Conf;
 
   if ( $options{thirdparty} ) {
-    # still a kludge, but it gets the job done
-    # and the 'cardtype' semantics don't really apply to thirdparty
-    # gateways because we have to choose a gateway without ever 
-    # seeing the card number
-    my $gatewaynum =
-      $conf->config('selfservice-payment_gateway', $self->agentnum);
-    my $gateway;
-    $gateway = FS::payment_gateway->by_key($gatewaynum) if $gatewaynum;
-    return $gateway if $gateway;
-
-    # a little less kludgey than the above, and allows PayPal to coexist 
-    # with credit card gateways
+
+    # allows PayPal to coexist with credit card gateways
     my $is_paypal = { op => '!=', value => 'PayPal' };
     if ( uc($options{method}) eq 'PAYPAL' ) {
       $is_paypal = 'PayPal';
     }
 
-    $gateway = qsearchs({
+    my $gateway = qsearchs({
         table     => 'payment_gateway',
         addl_from => ' JOIN agent_payment_gateway USING (gatewaynum) ',
         hashref   => {
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index e7226fe..fcc573e 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -226,14 +226,6 @@ sub _bop_recurring_billing {
 sub _payment_gateway {
   my ($self, $options) = @_;
 
-  if ( $options->{'selfservice'} ) {
-    my $gatewaynum = FS::Conf->new->config('selfservice-payment_gateway');
-    if ( $gatewaynum ) {
-      return $options->{payment_gateway} ||= 
-          qsearchs('payment_gateway', { gatewaynum => $gatewaynum });
-    }
-  }
-
   if ( $options->{'fake_gatewaynum'} ) {
 	$options->{payment_gateway} =
 	    qsearchs('payment_gateway',

commit 9026d337602fd1f9c463de2a46db4f27c0d429f9
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Nov 8 02:48:09 2016 -0600

    71513: Card tokenization [tokenize for unsaved cards, bug fix to saving from cust_main]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 407b9ca..e7226fe 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -375,13 +375,14 @@ sub _bop_content {
   \%content;
 }
 
+# updates payinfo and cust_payby options with token from transaction
 sub _tokenize_card {
   my ($self,$transaction,$options) = @_;
   if ( $transaction->can('card_token') 
        and $transaction->card_token 
        and !$self->tokenized($options->{'payinfo'})
   ) {
-    $options->{'payinfo'} = $transaction->card_token; #for creating cust_pay
+    $options->{'payinfo'} = $transaction->card_token;
     $options->{'cust_payby'}->payinfo($transaction->card_token) if $options->{'cust_payby'};
     return $transaction->card_token;
   }
@@ -418,6 +419,19 @@ sub realtime_bop {
   # set fields from passed cust_payby
   $self->_bop_cust_payby_options(\%options);
 
+  # possibly run a separate transaction to tokenize card number,
+  #   so that we never store tokenized card info in cust_pay_pending
+  if (!$self->tokenized($options{'payinfo'})) {
+    my $token_error = $self->realtime_tokenize(\%options);
+    return $token_error if $token_error;
+    # in theory, all cust_payby will be tokenized during original save,
+    # so we shouldn't get here with opt cust_payby...but just in case...
+    if ($options{'cust_payby'}) {
+      $token_error = $options{'cust_payby'}->replace;
+      return $token_error if $token_error;
+    }
+  }
+
   ### 
   # optional credit card surcharge
   ###
@@ -801,6 +815,8 @@ sub realtime_bop {
   # Tokenize
   ###
 
+  # This block will only run if the B::OP module supports card_token but not the Tokenize transaction;
+  #   if that never happens, we should get rid of it (as it has the potential to store real card numbers on error)
   if (my $card_token = $self->_tokenize_card($transaction,\%options)) {
     # cpp will be replaced in _realtime_bop_result
     $cust_pay_pending->payinfo($card_token);
@@ -906,7 +922,7 @@ sub _realtime_bop_result {
     or return "no payment gateway in arguments to _realtime_bop_result";
 
   $cust_pay_pending->status($transaction->is_success() ? 'captured' : 'declined');
-  my $cpp_captured_err = $cust_pay_pending->replace; #also saves tokenization
+  my $cpp_captured_err = $cust_pay_pending->replace; #also saves post-transaction tokenization, if that happens
   return $cpp_captured_err if $cpp_captured_err;
 
   if ( $transaction->is_success() ) {
@@ -1755,6 +1771,15 @@ sub realtime_verify_bop {
   return "No cust_payby" unless $options{'cust_payby'};
   $self->_bop_cust_payby_options(\%options);
 
+  # possibly run a separate transaction to tokenize card number,
+  #   so that we never store tokenized card info in cust_pay_pending
+  if (!$self->tokenized($options{'payinfo'})) {
+    my $token_error = $self->realtime_tokenize(\%options);
+    return $token_error if $token_error;
+    #important that we not replace cust_payby here,
+    #because cust_payby->replace uses realtime_verify_bop!
+  }
+
   ###
   # select a gateway
   ###
@@ -2113,13 +2138,15 @@ sub realtime_verify_bop {
   # Tokenize
   ###
 
-  #important that we not replace cust_payby here,
-  #because cust_payby->replace uses realtime_verify_bop!
+  # This block will only run if the B::OP module supports card_token but not the Tokenize transaction;
+  #   if that never happens, we should get rid of it (as it has the potential to store real card numbers on error)
   if (my $card_token = $self->_tokenize_card($transaction,\%options)) {
     $cust_pay_pending->payinfo($card_token);
     my $cpp_token_err = $cust_pay_pending->replace;
-    #this leaves real card number in cust_payby, but can't do much else if cust_payby won't replace
+    #this leaves real card number in cust_pay_pending, but can't do much else if cpp won't replace
     return $cpp_token_err if $cpp_token_err;
+    #important that we not replace cust_payby here,
+    #because cust_payby->replace uses realtime_verify_bop!
   }
 
   ###
@@ -2134,20 +2161,25 @@ sub realtime_verify_bop {
 
 =item realtime_tokenize [ OPTION => VALUE ... ]
 
-If possible, runs a tokenize transaction.
+If possible and necessary, runs a tokenize transaction.
 In order to be possible, a credit card cust_payby record
 must be passed and a Business::OnlinePayment gateway capable
 of Tokenize transactions must be configured for this user.
+Is only necessary if payinfo is not yet tokenized.
 
 Returns the empty string if the authorization was sucessful
-or was not possible (thus allowing this to be safely called with
+or was not possible/necessary (thus allowing this to be safely called with
 non-tokenizable records/gateways, without having to perform separate tests),
 or an error message otherwise.
 
-Option I<cust_payby> should be passed, even if it's not yet been inserted.
+Option I<cust_payby> may be passed, even if it's not yet been inserted.
 Object will be tokenized if possible, but that change will not be
 updated in database (must be inserted/replaced afterwards.)
 
+Otherwise, options I<method>, I<payinfo> and other cust_payby fields
+may be passed.  If options are passed as a hashref, I<payinfo>
+will be updated as appropriate in the passed hashref.
+
 =cut
 
 sub realtime_tokenize {
@@ -2157,14 +2189,16 @@ sub realtime_tokenize {
   my $log = FS::Log->new('FS::cust_main::Billing_Realtime::realtime_tokenize');
 
   my %options = ();
+  my $outoptions; #for returning cust_payby/payinfo
   if (ref($_[0]) eq 'HASH') {
     %options = %{$_[0]};
+    $outoptions = $_[0];
   } else {
     %options = @_;
+    $outoptions = \%options;
   }
 
   # set fields from passed cust_payby
-  return "No cust_payby" unless $options{'cust_payby'};
   $self->_bop_cust_payby_options(\%options);
   return '' unless $options{method} eq 'CC';
   return '' if $self->tokenized($options{payinfo}); #already tokenized
@@ -2186,7 +2220,6 @@ sub realtime_tokenize {
   # check for tokenize ability
   ###
 
-  # just create transaction now, so it loads gateway_module
   my $transaction = new $namespace( $payment_gateway->gateway_module,
                                     $self->_bop_options(\%options),
                                   );
@@ -2265,11 +2298,11 @@ sub realtime_tokenize {
 
     #important that we not replace cust_payby here, 
     #because cust_payby->replace uses realtime_tokenize!
-    $self->_tokenize_card($transaction,\%options);
+    $self->_tokenize_card($transaction,$outoptions);
 
   } else {
 
-    $error = $transaction->error_message || 'Unknown error';
+    $error = $transaction->error_message || 'Unknown error when tokenizing card';
 
   }
 
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 6982834..dfcce2f 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -67,7 +67,7 @@ sub payinfo {
   my($self,$payinfo) = @_;
 
   if ( defined($payinfo) ) {
-    $self->paymask($self->mask_payinfo) unless $self->paymask || $self->tokenized; #make sure old mask is set
+    $self->paymask($self->mask_payinfo) unless $self->getfield('paymask') || $self->tokenized; #make sure old mask is set
     $self->setfield('payinfo', $payinfo);
     $self->paymask($self->mask_payinfo) unless $self->tokenized($payinfo); #remask unless tokenizing
   } else {

commit 1185daff43389fe53ad43e84962329a63d31523e
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Sat Nov 5 01:20:35 2016 -0500

    71513: Card tokenization [bug fixes to previous checkpoint]

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 09e2dfa..407b9ca 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -376,44 +376,16 @@ sub _bop_content {
 }
 
 sub _tokenize_card {
-  my ($self,$transaction,$options,$log,%opt) = @_;
-  # options is for entire process, so we can update payinfo
-  # opt is just for this call, only key is replace
-
-  my $cust_payby = $options->{'cust_payby'};
-  if ( $cust_payby
-       and $transaction->can('card_token') 
+  my ($self,$transaction,$options) = @_;
+  if ( $transaction->can('card_token') 
        and $transaction->card_token 
-       and !$cust_payby->tokenized #not already tokenized
+       and !$self->tokenized($options->{'payinfo'})
   ) {
-
-    $options->{'payinfo'} = $transaction->card_token;
-    $cust_payby->payinfo($transaction->card_token);
-
-    my $error;
-    $error = $cust_payby->replace if $opt{'replace'};
-    if ( $error ) {
-      $log->error('Error storing token for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum.': '.$error);
-      return $error;
-    } else {
-      $log->debug('Tokenized card for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum);
-      return '';
-    }
-
+    $options->{'payinfo'} = $transaction->card_token; #for creating cust_pay
+    $options->{'cust_payby'}->payinfo($transaction->card_token) if $options->{'cust_payby'};
+    return $transaction->card_token;
   }
-
-}
-
-# only store payinfo in cust_pay/cust_pay_pending
-# if it's a tokenized card or if processor requires card for void
-sub _cust_pay_opts {
-  my ($self,$payby,$payinfo,$transaction) = @_;
-  ( (($payby eq 'CARD') && $self->tokenized($payinfo))
-    || (($payby eq 'CARD') && $transaction->info('CC_void_requires_card'))
-    || (($payby eq 'CHEK') && $transaction->info('ECHECK_void_requires_account'))
-  )
-    ? ('payinfo' => $payinfo)
-    : ();
+  return '';
 }
 
 my %bop_method2payby = (
@@ -681,15 +653,12 @@ sub realtime_bop {
 
   #okay, good to go, if we're a duplicate, cust_pay_pending will kick us out
 
-  my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
-                                  );
-
   my $cust_pay_pending = new FS::cust_pay_pending {
     'custnum'           => $self->custnum,
     'paid'              => $options{amount},
     '_date'             => '',
     'payby'             => $bop_method2payby{$options{method}},
+    'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
     'paydate'           => $paydate,
     'recurring_billing' => $content{recurring_billing},
@@ -698,7 +667,6 @@ sub realtime_bop {
     'gatewaynum'        => $payment_gateway->gatewaynum || '',
     'session_id'        => $options{session_id} || '',
     'jobnum'            => $options{depend_jobnum} || '',
-    $self->_cust_pay_opts($options{payinfo},$transaction),
   };
   $cust_pay_pending->payunique( $options{payunique} )
     if defined($options{payunique}) && length($options{payunique});
@@ -715,6 +683,10 @@ sub realtime_bop {
   my( $action1, $action2 ) =
     split( /\s*\,\s*/, $payment_gateway->gateway_action );
 
+  my $transaction = new $namespace( $payment_gateway->gateway_module,
+                                    $self->_bop_options(\%options),
+                                  );
+
   $transaction->content(
     'type'           => $options{method},
     $self->_bop_auth(\%options),          
@@ -819,6 +791,8 @@ sub realtime_bop {
   ) {
     my $error = $self->remove_cvv_from_cust_payby($options{payinfo});
     if ( $error ) {
+      $log->critical('Error removing cvv for cust '.$self->custnum.': '.$error);
+      #not returning error, should at least attempt to handle results of an otherwise valid transaction
       warn "WARNING: error removing cvv: $error\n";
     }
   }
@@ -827,8 +801,15 @@ sub realtime_bop {
   # Tokenize
   ###
 
-  my $error = $self->_tokenize_card($transaction,\%options,$log,'replace' => 1);
-  return $error if $error;
+  if (my $card_token = $self->_tokenize_card($transaction,\%options)) {
+    # cpp will be replaced in _realtime_bop_result
+    $cust_pay_pending->payinfo($card_token);
+    if ($options{'cust_payby'} and my $error = $options{'cust_payby'}->replace) {
+      $log->critical('Error storing token for cust '.$self->custnum.', cust_payby '.$options{'cust_payby'}->custpaybynum.': '.$error);
+      #not returning error, should at least attempt to handle results of an otherwise valid transaction
+      #this leaves real card number in cust_payby, but can't do much else if cust_payby won't replace
+    }
+  }
 
   ###
   # result handling
@@ -925,7 +906,7 @@ sub _realtime_bop_result {
     or return "no payment gateway in arguments to _realtime_bop_result";
 
   $cust_pay_pending->status($transaction->is_success() ? 'captured' : 'declined');
-  my $cpp_captured_err = $cust_pay_pending->replace;
+  my $cpp_captured_err = $cust_pay_pending->replace; #also saves tokenization
   return $cpp_captured_err if $cpp_captured_err;
 
   if ( $transaction->is_success() ) {
@@ -939,6 +920,7 @@ sub _realtime_bop_result {
        'paid'     => $cust_pay_pending->paid,
        '_date'    => '',
        'payby'    => $cust_pay_pending->payby,
+       'payinfo'  => $options{'payinfo'},
        'paymask'  => $options{'paymask'} || $cust_pay_pending->paymask,
        'paydate'  => $cust_pay_pending->paydate,
        'pkgnum'   => $cust_pay_pending->pkgnum,
@@ -948,7 +930,6 @@ sub _realtime_bop_result {
        'auth'           => $transaction->authorization,
        'order_number'   => $order_number || '',
        'no_auto_apply'  => $options{'no_auto_apply'} ? 'Y' : '',
-       $self->_cust_pay_opts($options{payinfo},$transaction),
     } );
     #doesn't hurt to know, even though the dup check is in cust_pay_pending now
     $cust_pay->payunique( $options{payunique} )
@@ -1854,9 +1835,7 @@ sub realtime_verify_bop {
   ###
 
   my $error;
-  my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
-                                  ); #need this back so we can do _tokenize_card
+  my $transaction; #need this back so we can do _tokenize_card
 
   # don't mutex the customer here, because they might be uncommitted. and
   # this is only verification. it doesn't matter if they have other
@@ -1867,13 +1846,13 @@ sub realtime_verify_bop {
     'paid'              => '1.00',
     '_date'             => '',
     'payby'             => $bop_method2payby{'CC'},
+    'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
     'paydate'           => $paydate,
     'pkgnum'            => $options{'pkgnum'},
     'status'            => 'new',
     'gatewaynum'        => $payment_gateway->gatewaynum || '',
     'session_id'        => $options{session_id} || '',
-    $self->_cust_pay_opts($options{payinfo},$transaction),
   };
   $cust_pay_pending->payunique( $options{payunique} )
     if defined($options{payunique}) && length($options{payunique});
@@ -1904,6 +1883,10 @@ sub realtime_verify_bop {
       if $DEBUG > 1;
     warn Dumper($cust_pay_pending) if $DEBUG > 2;
 
+    $transaction = new $namespace( $payment_gateway->gateway_module,
+                                   $self->_bop_options(\%options),
+                                    );
+
     $transaction->content(
       'type'           => 'CC',
       $self->_bop_auth(\%options),          
@@ -2122,12 +2105,22 @@ sub realtime_verify_bop {
   }
 
   ###
+  # remove paycvv here?  need to find out if a reversed auth
+  #   counts as an initial transaction for paycvv retention requirements
+  ###
+
+  ###
   # Tokenize
   ###
 
-  #important that we not pass replace option here,
+  #important that we not replace cust_payby here,
   #because cust_payby->replace uses realtime_verify_bop!
-  $self->_tokenize_card($transaction,\%options,$log);
+  if (my $card_token = $self->_tokenize_card($transaction,\%options)) {
+    $cust_pay_pending->payinfo($card_token);
+    my $cpp_token_err = $cust_pay_pending->replace;
+    #this leaves real card number in cust_payby, but can't do much else if cust_payby won't replace
+    return $cpp_token_err if $cpp_token_err;
+  }
 
   ###
   # result handling
@@ -2263,9 +2256,16 @@ sub realtime_tokenize {
 
   if ( $transaction->card_token() ) { # no is_success flag
 
-    #important that we not pass replace option here, 
+    # realtime_tokenize should not clear paycvv at this time.  it might be
+    # needed for the first transaction, and a tokenize isn't actually a
+    # transaction that hits the gateway.  at some point in the future, card
+    # fortress should take on the "store paycvv until first transaction"
+    # functionality and we should fix this in freeside, but i that's a bigger
+    # project for another time.
+
+    #important that we not replace cust_payby here, 
     #because cust_payby->replace uses realtime_tokenize!
-    $self->_tokenize_card($transaction,\%options,$log);
+    $self->_tokenize_card($transaction,\%options);
 
   } else {
 
@@ -2277,10 +2277,19 @@ sub realtime_tokenize {
 
 }
 
+
+=item tokenized PAYINFO
+
+Convenience wrapper for L<FS::payinfo_Mixin/tokenized>
+
+PAYINFO is required
+
+=cut
+
 sub tokenized {
   my $this = shift;
   my $payinfo = shift;
-  $payinfo =~ /^99\d{14}$/;
+  FS::cust_pay->tokenized($payinfo);
 }
 
 =back
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index a0a2cbc..6982834 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -67,7 +67,7 @@ sub payinfo {
   my($self,$payinfo) = @_;
 
   if ( defined($payinfo) ) {
-    $self->paymask($self->mask_payinfo) unless $self->tokenized; #make sure old mask is set
+    $self->paymask($self->mask_payinfo) unless $self->paymask || $self->tokenized; #make sure old mask is set
     $self->setfield('payinfo', $payinfo);
     $self->paymask($self->mask_payinfo) unless $self->tokenized($payinfo); #remask unless tokenizing
   } else {
@@ -454,12 +454,17 @@ sub process_set_cardtype {
   }
 }
 
+=item tokenized [ PAYINFO ]
+
+Returns true if object payinfo is tokenized
+
+Optionally, an arbitrary payby and payinfo can be passed.
+
+=cut
+
 sub tokenized {
   my $self = shift;
   my $payinfo = scalar(@_) ? shift : $self->payinfo;
-  ## or just $self->cust_main->tokenized($payinfo) ??
-  ##   everything that currently uses this mixin is linked to cust_main,
-  ##   but just in case, false laziness w/ FS::cust_main::Billing_Realtime
   $payinfo =~ /^99\d{14}$/;
 }
 
diff --git a/httemplate/misc/process/payment.cgi b/httemplate/misc/process/payment.cgi
index 84687f0..1532605 100644
--- a/httemplate/misc/process/payment.cgi
+++ b/httemplate/misc/process/payment.cgi
@@ -193,6 +193,11 @@ if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
 
     errorpage("error saving info, payment not processed: $error")
       if $error;	
+
+  } elsif ( $payby eq 'CARD' ) { # not saving
+
+    $paymask = FS::payinfo_Mixin->mask_payinfo('CARD',$payinfo); # for untokenized but tokenizable payinfo
+
   }
 
 }

commit 16498ac263bf5f3e90e23b866706ada768486f40
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Sat Oct 29 12:02:31 2016 -0500

    71513: Card tokenization [checkpoint, not ready for backport]

diff --git a/FS/FS/ClientAPI/MyAccount.pm b/FS/FS/ClientAPI/MyAccount.pm
index 7c17ae3..091d6ac 100644
--- a/FS/FS/ClientAPI/MyAccount.pm
+++ b/FS/FS/ClientAPI/MyAccount.pm
@@ -1022,7 +1022,7 @@ sub validate_payment {
     validate($payinfo)
       or return { 'error' => gettext('invalid_card') }; # . ": ". $self->payinfo
     return { 'error' => gettext('unknown_card_type') }
-      if $payinfo !~ /^99\d{14}$/ && cardtype($payinfo) eq "Unknown";
+      if !$cust_main->tokenized($payinfo) && cardtype($payinfo) eq "Unknown";
 
     if ( length($p->{'paycvv'}) && $p->{'paycvv'} !~ /^\s*$/ ) {
       if ( cardtype($payinfo) eq 'American Express card' ) {
diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index eac6c75..a2c0ee8 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -2128,7 +2128,7 @@ sub check_payinfo_cardtype {
   my $payinfo = $self->payinfo;
   $payinfo =~ s/\D//g;
 
-  return '' if $payinfo =~ /^99\d{14}$/; #token
+  return '' if $self->tokenized($payinfo); #token
 
   my %bop_card_types = map { $_=>1 } values %{ card_types() };
   my $cardtype = cardtype($payinfo);
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index 81b00aa..09e2dfa 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -376,14 +376,18 @@ sub _bop_content {
 }
 
 sub _tokenize_card {
-  my ($self,$transaction,$cust_payby,$log,%opt) = @_;
+  my ($self,$transaction,$options,$log,%opt) = @_;
+  # options is for entire process, so we can update payinfo
+  # opt is just for this call, only key is replace
 
+  my $cust_payby = $options->{'cust_payby'};
   if ( $cust_payby
        and $transaction->can('card_token') 
        and $transaction->card_token 
-       and $cust_payby->payinfo !~ /^99\d{14}$/ #not already tokenized
+       and !$cust_payby->tokenized #not already tokenized
   ) {
 
+    $options->{'payinfo'} = $transaction->card_token;
     $cust_payby->payinfo($transaction->card_token);
 
     my $error;
@@ -400,6 +404,18 @@ sub _tokenize_card {
 
 }
 
+# only store payinfo in cust_pay/cust_pay_pending
+# if it's a tokenized card or if processor requires card for void
+sub _cust_pay_opts {
+  my ($self,$payby,$payinfo,$transaction) = @_;
+  ( (($payby eq 'CARD') && $self->tokenized($payinfo))
+    || (($payby eq 'CARD') && $transaction->info('CC_void_requires_card'))
+    || (($payby eq 'CHEK') && $transaction->info('ECHECK_void_requires_account'))
+  )
+    ? ('payinfo' => $payinfo)
+    : ();
+}
+
 my %bop_method2payby = (
   'CC'     => 'CARD',
   'ECHECK' => 'CHEK',
@@ -665,12 +681,15 @@ sub realtime_bop {
 
   #okay, good to go, if we're a duplicate, cust_pay_pending will kick us out
 
+  my $transaction = new $namespace( $payment_gateway->gateway_module,
+                                    $self->_bop_options(\%options),
+                                  );
+
   my $cust_pay_pending = new FS::cust_pay_pending {
     'custnum'           => $self->custnum,
     'paid'              => $options{amount},
     '_date'             => '',
     'payby'             => $bop_method2payby{$options{method}},
-    'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
     'paydate'           => $paydate,
     'recurring_billing' => $content{recurring_billing},
@@ -679,6 +698,7 @@ sub realtime_bop {
     'gatewaynum'        => $payment_gateway->gatewaynum || '',
     'session_id'        => $options{session_id} || '',
     'jobnum'            => $options{depend_jobnum} || '',
+    $self->_cust_pay_opts($options{payinfo},$transaction),
   };
   $cust_pay_pending->payunique( $options{payunique} )
     if defined($options{payunique}) && length($options{payunique});
@@ -695,10 +715,6 @@ sub realtime_bop {
   my( $action1, $action2 ) =
     split( /\s*\,\s*/, $payment_gateway->gateway_action );
 
-  my $transaction = new $namespace( $payment_gateway->gateway_module,
-                                    $self->_bop_options(\%options),
-                                  );
-
   $transaction->content(
     'type'           => $options{method},
     $self->_bop_auth(\%options),          
@@ -811,7 +827,7 @@ sub realtime_bop {
   # Tokenize
   ###
 
-  my $error = $self->_tokenize_card($transaction,$options{'cust_payby'},$log,'replace' => 1);
+  my $error = $self->_tokenize_card($transaction,\%options,$log,'replace' => 1);
   return $error if $error;
 
   ###
@@ -849,9 +865,7 @@ sub fake_bop {
      'paid'     => $options{amount},
      '_date'    => '',
      'payby'    => $bop_method2payby{$options{method}},
-     #'payinfo'  => $payinfo,
      'payinfo'  => '4111111111111111',
-     #'paydate'  => $paydate,
      'paydate'  => '2012-05-01',
      'processor'      => 'FakeProcessor',
      'auth'           => '54',
@@ -925,7 +939,6 @@ sub _realtime_bop_result {
        'paid'     => $cust_pay_pending->paid,
        '_date'    => '',
        'payby'    => $cust_pay_pending->payby,
-       'payinfo'  => $options{'payinfo'},
        'paymask'  => $options{'paymask'} || $cust_pay_pending->paymask,
        'paydate'  => $cust_pay_pending->paydate,
        'pkgnum'   => $cust_pay_pending->pkgnum,
@@ -935,6 +948,7 @@ sub _realtime_bop_result {
        'auth'           => $transaction->authorization,
        'order_number'   => $order_number || '',
        'no_auto_apply'  => $options{'no_auto_apply'} ? 'Y' : '',
+       $self->_cust_pay_opts($options{payinfo},$transaction),
     } );
     #doesn't hurt to know, even though the dup check is in cust_pay_pending now
     $cust_pay->payunique( $options{payunique} )
@@ -1840,7 +1854,9 @@ sub realtime_verify_bop {
   ###
 
   my $error;
-  my $transaction; #need this back so we can do _tokenize_card
+  my $transaction = new $namespace( $payment_gateway->gateway_module,
+                                    $self->_bop_options(\%options),
+                                  ); #need this back so we can do _tokenize_card
 
   # don't mutex the customer here, because they might be uncommitted. and
   # this is only verification. it doesn't matter if they have other
@@ -1851,13 +1867,13 @@ sub realtime_verify_bop {
     'paid'              => '1.00',
     '_date'             => '',
     'payby'             => $bop_method2payby{'CC'},
-    'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
     'paydate'           => $paydate,
     'pkgnum'            => $options{'pkgnum'},
     'status'            => 'new',
     'gatewaynum'        => $payment_gateway->gatewaynum || '',
     'session_id'        => $options{session_id} || '',
+    $self->_cust_pay_opts($options{payinfo},$transaction),
   };
   $cust_pay_pending->payunique( $options{payunique} )
     if defined($options{payunique}) && length($options{payunique});
@@ -1888,10 +1904,6 @@ sub realtime_verify_bop {
       if $DEBUG > 1;
     warn Dumper($cust_pay_pending) if $DEBUG > 2;
 
-    $transaction = new $namespace( $payment_gateway->gateway_module,
-                                   $self->_bop_options(\%options),
-                                    );
-
     $transaction->content(
       'type'           => 'CC',
       $self->_bop_auth(\%options),          
@@ -2115,7 +2127,7 @@ sub realtime_verify_bop {
 
   #important that we not pass replace option here,
   #because cust_payby->replace uses realtime_verify_bop!
-  $self->_tokenize_card($transaction,$options{'cust_payby'},$log);
+  $self->_tokenize_card($transaction,\%options,$log);
 
   ###
   # result handling
@@ -2162,7 +2174,7 @@ sub realtime_tokenize {
   return "No cust_payby" unless $options{'cust_payby'};
   $self->_bop_cust_payby_options(\%options);
   return '' unless $options{method} eq 'CC';
-  return '' if $options{payinfo} =~ /^99\d{14}$/; #already tokenized
+  return '' if $self->tokenized($options{payinfo}); #already tokenized
 
   ###
   # select a gateway
@@ -2253,7 +2265,7 @@ sub realtime_tokenize {
 
     #important that we not pass replace option here, 
     #because cust_payby->replace uses realtime_tokenize!
-    $self->_tokenize_card($transaction,$options{'cust_payby'},$log);
+    $self->_tokenize_card($transaction,\%options,$log);
 
   } else {
 
@@ -2265,6 +2277,12 @@ sub realtime_tokenize {
 
 }
 
+sub tokenized {
+  my $this = shift;
+  my $payinfo = shift;
+  $payinfo =~ /^99\d{14}$/;
+}
+
 =back
 
 =head1 BUGS
diff --git a/FS/FS/cust_payby.pm b/FS/FS/cust_payby.pm
index 626fc9f..53608cf 100644
--- a/FS/FS/cust_payby.pm
+++ b/FS/FS/cust_payby.pm
@@ -276,7 +276,7 @@ sub replace {
 
   if ( $self->payby =~ /^(CARD|CHEK)$/
        && ( ( $self->get('payinfo') ne $old->get('payinfo')
-              && $self->get('payinfo') !~ /^99\d{14}$/ 
+              && !$self->tokenized 
             )
             || grep { $self->get($_) ne $old->get($_) } qw(paydate payname)
           )
@@ -357,7 +357,7 @@ sub check {
       or return gettext('invalid_card'); # . ": ". $self->payinfo;
 
     my $cardtype = cardtype($payinfo);
-    $cardtype = 'Tokenized' if $self->payinfo =~ /^99\d{14}$/; #token
+    $cardtype = 'Tokenized' if $self->tokenized; #token
     
     return gettext('unknown_card_type') if $cardtype eq "Unknown";
     
@@ -546,7 +546,7 @@ sub check_payinfo_cardtype {
   my $payinfo = $self->payinfo;
   $payinfo =~ s/\D//g;
 
-  if ( $payinfo =~ /^99\d{14}$/ ) {
+  if ( $self->tokenized($payinfo) ) {
     $self->set('paycardtype', 'Tokenized');
     return '';
   }
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 3a32ad5..a0a2cbc 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -67,9 +67,9 @@ sub payinfo {
   my($self,$payinfo) = @_;
 
   if ( defined($payinfo) ) {
-    $self->paymask($self->mask_payinfo) unless $self->payinfo =~ /^99\d{14}$/; #make sure old mask is set
+    $self->paymask($self->mask_payinfo) unless $self->tokenized; #make sure old mask is set
     $self->setfield('payinfo', $payinfo);
-    $self->paymask($self->mask_payinfo) unless $payinfo =~ /^99\d{14}$/; #remask unless tokenizing
+    $self->paymask($self->mask_payinfo) unless $self->tokenized($payinfo); #remask unless tokenizing
   } else {
     $self->getfield('payinfo');
   }
@@ -130,7 +130,7 @@ sub mask_payinfo {
   # Check to see if it's encrypted...
   if ( ref($self) && $self->is_encrypted($payinfo) ) {
     return 'N/A';
-  } elsif ( $payinfo =~ /^99\d{14}$/ || $payinfo eq 'N/A' ) { #token
+  } elsif ( $self->tokenized($payinfo) || $payinfo eq 'N/A' ) { #token
     return 'N/A (tokenized)'; #?
   } else { # if not, mask it...
 
@@ -198,7 +198,7 @@ sub payinfo_check {
 
     my $payinfo = $self->payinfo;
     my $cardtype = cardtype($payinfo);
-    $cardtype = 'Tokenized' if $payinfo =~ /^99\d{14}$/;
+    $cardtype = 'Tokenized' if $self->tokenized;
     $self->set('paycardtype', $cardtype);
 
     if ( $ignore_masked_payinfo and $self->mask_payinfo eq $self->payinfo ) {
@@ -233,6 +233,7 @@ sub payinfo_check {
     }
   }
 
+  return '';
 }
 
 =item payby_payinfo_pretty [ LOCALE ]
@@ -453,6 +454,15 @@ sub process_set_cardtype {
   }
 }
 
+sub tokenized {
+  my $self = shift;
+  my $payinfo = scalar(@_) ? shift : $self->payinfo;
+  ## or just $self->cust_main->tokenized($payinfo) ??
+  ##   everything that currently uses this mixin is linked to cust_main,
+  ##   but just in case, false laziness w/ FS::cust_main::Billing_Realtime
+  $payinfo =~ /^99\d{14}$/;
+}
+
 =back
 
 =head1 BUGS
diff --git a/httemplate/misc/process/payment.cgi b/httemplate/misc/process/payment.cgi
index 74ca734..84687f0 100644
--- a/httemplate/misc/process/payment.cgi
+++ b/httemplate/misc/process/payment.cgi
@@ -135,7 +135,7 @@ if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
     validate($payinfo)
       or errorpage(gettext('invalid_card'));
 
-    unless ( $payinfo =~ /^99\d{14}$/ ) { #token
+    unless ( $cust_main->tokenized($payinfo) ) { #token
 
       my $cardtype = cardtype($payinfo);
 

commit 80542a7f5c52ac2f631adc82d0e4326554200793
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Tue Oct 11 20:43:13 2016 -0500

    71513: Card tokenization in v4+

diff --git a/FS/FS/cust_main.pm b/FS/FS/cust_main.pm
index e1f73bf..eac6c75 100644
--- a/FS/FS/cust_main.pm
+++ b/FS/FS/cust_main.pm
@@ -4679,6 +4679,10 @@ CHEK only
 
 CHEK only
 
+=item saved_cust_payby
+
+scalar reference, for returning saved object
+
 =back
 
 =cut
@@ -4875,6 +4879,9 @@ PAYBYLOOP:
     return $error;
   }
 
+  ${$opt{'saved_cust_payby'}} = $new
+    if $opt{'saved_cust_payby'};
+
   $dbh->commit or die $dbh->errstr if $oldAutoCommit;
   '';
 
diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index cb7299b..81b00aa 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -111,6 +111,8 @@ I<depend_jobnum> allows payment capture to unlock export jobs
 
 =cut
 
+# Currently only used by ClientAPI
+# NOT 4.x COMPATIBLE (see below)
 sub realtime_collect {
   my( $self, %options ) = @_;
 
@@ -124,6 +126,7 @@ sub realtime_collect {
   $options{amount} = $self->balance unless exists( $options{amount} );
   return '' unless $options{amount} > 0;
 
+  #### NOT 4.x COMPATIBLE
   $options{method} = FS::payby->payby2bop($self->payby)
     unless exists( $options{method} );
 
@@ -137,16 +140,14 @@ Runs a realtime credit card or ACH (electronic check) transaction
 via a Business::OnlinePayment realtime gateway.  See
 L<http://420.am/business-onlinepayment> for supported gateways.
 
-Required arguments in the hashref are I<method>, and I<amount>
+Required arguments in the hashref are I<amount> and either
+I<cust_payby> or I<method>, I<payinfo> and (as applicable for method)
+I<payname>, I<address1>, I<address2>, I<city>, I<state>, I<zip> and I<paydate>.
 
 Available methods are: I<CC>, I<ECHECK>, or I<PAYPAL>
 
 Available optional arguments are: I<description>, I<invnum>, I<apply>, I<quiet>, I<paynum_ref>, I<payunique>, I<session_id>
 
-The additional options I<payname>, I<address1>, I<address2>, I<city>, I<state>,
-I<zip>, I<payinfo> and I<paydate> are also available.  Any of these options,
-if set, will override the value from the customer record.
-
 I<description> is a free-text field passed to the gateway.  It defaults to
 the value defined by the business-onlinepayment-description configuration
 option, or "Internet services" if that is unset.
@@ -279,11 +280,6 @@ sub _bop_defaults {
     }
   }
 
-  unless ( exists( $options->{'payinfo'} ) ) {
-    $options->{'payinfo'} = $self->payinfo;
-    $options->{'paymask'} = $self->paymask;
-  }
-
   # Default invoice number if the customer has exactly one open invoice.
   unless ( $options->{'invnum'} || $options->{'no_invnum'} ) {
     $options->{'invnum'} = '';
@@ -291,14 +287,50 @@ sub _bop_defaults {
     $options->{'invnum'} = $open[0]->invnum if scalar(@open) == 1;
   }
 
-  $options->{payname} = $self->payname unless exists( $options->{payname} );
+}
+
+sub _bop_cust_payby_options {
+  my ($self,$options) = @_;
+  my $cust_payby = $options->{'cust_payby'};
+  if ($cust_payby) {
+
+    $options->{'method'} = FS::payby->payby2bop( $cust_payby->payby );
+
+    if ($cust_payby->payby =~ /^(CARD|DCRD)$/) {
+      # false laziness with cust_payby->check
+      #   which might not have been run yet
+      my( $m, $y );
+      if ( $cust_payby->paydate =~ /^(\d{1,2})[\/\-](\d{2}(\d{2})?)$/ ) {
+        ( $m, $y ) = ( $1, length($2) == 4 ? $2 : "20$2" );
+      } elsif ( $cust_payby->paydate =~ /^19(\d{2})[\/\-](\d{1,2})[\/\-]\d+$/ ) {
+        ( $m, $y ) = ( $2, "19$1" );
+      } elsif ( $cust_payby->paydate =~ /^(20)?(\d{2})[\/\-](\d{1,2})[\/\-]\d+$/ ) {
+        ( $m, $y ) = ( $3, "20$2" );
+      } else {
+        return "Illegal expiration date: ". $cust_payby->paydate;
+      }
+      $m = sprintf('%02d',$m);
+      $options->{paydate} = "$y-$m-01";
+    } else {
+      $options->{paydate} = '';
+    }
+
+    $options->{$_} = $cust_payby->$_() 
+      for qw( payinfo paycvv paymask paystart_month paystart_year 
+              payissue payname paystate paytype payip );
+
+    if ( $cust_payby->locationnum ) {
+      my $cust_location = $cust_payby->cust_location;
+      $options->{$_} = $cust_location->$_() for qw( address1 address2 city state zip );
+    }
+  }
 }
 
 sub _bop_content {
   my ($self, $options) = @_;
   my %content = ();
 
-  my $payip = exists($options->{'payip'}) ? $options->{'payip'} : $self->payip;
+  my $payip = $options->{'payip'};
   $content{customer_ip} = $payip if length($payip);
 
   $content{invoice_number} = $options->{'invnum'}
@@ -325,26 +357,14 @@ sub _bop_content {
 
   $content{name} = $payname;
 
-  $content{address} = exists($options->{'address1'})
-                        ? $options->{'address1'}
-                        : $self->address1;
-  my $address2 = exists($options->{'address2'})
-                   ? $options->{'address2'}
-                   : $self->address2;
+  $content{address} = $options->{'address1'};
+  my $address2 = $options->{'address2'};
   $content{address} .= ", ". $address2 if length($address2);
 
-  $content{city} = exists($options->{city})
-                     ? $options->{city}
-                     : $self->city;
-  $content{state} = exists($options->{state})
-                      ? $options->{state}
-                      : $self->state;
-  $content{zip} = exists($options->{zip})
-                    ? $options->{'zip'}
-                    : $self->zip;
-  $content{country} = exists($options->{country})
-                        ? $options->{country}
-                        : $self->country;
+  $content{city} = $options->{'city'};
+  $content{state} = $options->{'state'};
+  $content{zip} = $options->{'zip'};
+  $content{country} = $options->{'country'};
 
   $content{phone} = $self->daytime || $self->night;
 
@@ -356,28 +376,24 @@ sub _bop_content {
 }
 
 sub _tokenize_card {
-  my ($self,$transaction,$payinfo,$log) = @_;
+  my ($self,$transaction,$cust_payby,$log,%opt) = @_;
 
-  if ( $transaction->can('card_token') 
+  if ( $cust_payby
+       and $transaction->can('card_token') 
        and $transaction->card_token 
-       and $payinfo !~ /^99\d{14}$/ #not already tokenized
+       and $cust_payby->payinfo !~ /^99\d{14}$/ #not already tokenized
   ) {
 
-    my @cust_payby = $self->cust_payby('CARD','DCRD');
-    @cust_payby = grep { $payinfo == $_->payinfo } @cust_payby;
-    if (@cust_payby > 1) {
-      $log->error('Multiple matching card numbers for cust '.$self->custnum.', could not tokenize card');
-    } elsif (@cust_payby) {
-      my $cust_payby = $cust_payby[0];
-      $cust_payby->payinfo($transaction->card_token);
-      my $error = $cust_payby->replace;
-      if ( $error ) {
-        $log->error('Error storing token for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum.': '.$error);
-      } else {
-        $log->debug('Tokenized card for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum);
-      }
+    $cust_payby->payinfo($transaction->card_token);
+
+    my $error;
+    $error = $cust_payby->replace if $opt{'replace'};
+    if ( $error ) {
+      $log->error('Error storing token for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum.': '.$error);
+      return $error;
     } else {
-      $log->debug('No matching card numbers for cust '.$self->custnum.', could not tokenize card');
+      $log->debug('Tokenized card for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum);
+      return '';
     }
 
   }
@@ -411,6 +427,8 @@ sub realtime_bop {
     $options{amount} = $amount;
   }
 
+  # set fields from passed cust_payby
+  $self->_bop_cust_payby_options(\%options);
 
   ### 
   # optional credit card surcharge
@@ -450,6 +468,9 @@ sub realtime_bop {
 
   $self->_bop_defaults(\%options);
 
+  return "Missing payinfo"
+    unless $options{'payinfo'};
+
   ###
   # set trans_is_recur based on invnum if there is one
   ###
@@ -535,29 +556,19 @@ sub realtime_bop {
     if ( $options{method} eq 'CC' ) {
 
       $content{card_number} = $options{payinfo};
-      $paydate = exists($options{'paydate'})
-                      ? $options{'paydate'}
-                      : $self->paydate;
+      $paydate = $options{'paydate'};
       $paydate =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
       $content{expiration} = "$2/$1";
 
       $content{cvv2} = $options{'paycvv'}
         if length($options{'paycvv'});
 
-      my $paystart_month = exists($options{'paystart_month'})
-                             ? $options{'paystart_month'}
-                             : $self->paystart_month;
-
-      my $paystart_year  = exists($options{'paystart_year'})
-                             ? $options{'paystart_year'}
-                             : $self->paystart_year;
-
+      my $paystart_month = $options{'paystart_month'};
+      my $paystart_year  = $options{'paystart_year'};
       $content{card_start} = "$paystart_month/$paystart_year"
         if $paystart_month && $paystart_year;
 
-      my $payissue       = exists($options{'payissue'})
-                             ? $options{'payissue'}
-                             : $self->payissue;
+      my $payissue       = $options{'payissue'};
       $content{issue_number} = $payissue if $payissue;
 
       if ( $self->_bop_recurring_billing(
@@ -576,13 +587,8 @@ sub realtime_bop {
       ( $content{account_number}, $content{routing_code} ) =
         split('@', $options{payinfo});
       $content{bank_name} = $options{payname};
-      $content{bank_state} = exists($options{'paystate'})
-                               ? $options{'paystate'}
-                               : $self->getfield('paystate');
-      $content{account_type}=
-        (exists($options{'paytype'}) && $options{'paytype'})
-          ? uc($options{'paytype'})
-          : uc($self->getfield('paytype')) || 'PERSONAL CHECKING';
+      $content{bank_state} = $options{'paystate'};
+      $content{account_type}= uc($options{'paytype'}) || 'PERSONAL CHECKING';
 
       $content{company} = $self->company if $self->company;
 
@@ -805,7 +811,8 @@ sub realtime_bop {
   # Tokenize
   ###
 
-  $self->_tokenize_card($transaction,$options{'payinfo'},$log);
+  my $error = $self->_tokenize_card($transaction,$options{'cust_payby'},$log,'replace' => 1);
+  return $error if $error;
 
   ###
   # result handling
@@ -1721,21 +1728,14 @@ successful, immediatly reverses the authorization).
 Returns the empty string if the authorization was sucessful, or an error
 message otherwise.
 
-I<payinfo>
+Option I<cust_payby> should be passed, even if it's not yet been inserted.
+Object will be tokenized if possible, but that change will not be
+updated in database (must be inserted/replaced afterwards.)
 
-I<payname>
-
-I<paydate> specifies the expiration date for a credit card overriding the
-value from the customer record or the payment record. Specified as yyyy-mm-dd
-
-#The additional options I<address1>, I<address2>, I<city>, I<state>,
-#I<zip> are also available.  Any of these options,
-#if set, will override the value from the customer record.
+Currently only succeeds for Business::OnlinePayment CC transactions.
 
 =cut
 
-#Available methods are: I<CC> or I<ECHECK>
-
 #some false laziness w/realtime_bop and realtime_refund_bop, not enough to make
 #it worth merging but some useful small subs should be pulled out
 sub realtime_verify_bop {
@@ -1756,6 +1756,10 @@ sub realtime_verify_bop {
     warn "  $_ => $options{$_}\n" foreach keys %options;
   }
 
+  # set fields from passed cust_payby
+  return "No cust_payby" unless $options{'cust_payby'};
+  $self->_bop_cust_payby_options(\%options);
+
   ###
   # select a gateway
   ###
@@ -1802,43 +1806,33 @@ sub realtime_verify_bop {
     if ( $options{method} eq 'CC' ) {
 
       $content{card_number} = $options{payinfo};
-      $paydate = exists($options{'paydate'})
-                      ? $options{'paydate'}
-                      : $self->paydate;
+      $paydate = $options{'paydate'};
       $paydate =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
       $content{expiration} = "$2/$1";
 
       $content{cvv2} = $options{'paycvv'}
         if length($options{'paycvv'});
 
-      my $paystart_month = exists($options{'paystart_month'})
-                             ? $options{'paystart_month'}
-                             : $self->paystart_month;
-
-      my $paystart_year  = exists($options{'paystart_year'})
-                             ? $options{'paystart_year'}
-                             : $self->paystart_year;
+      my $paystart_month = $options{'paystart_month'};
+      my $paystart_year  = $options{'paystart_year'};
 
       $content{card_start} = "$paystart_month/$paystart_year"
         if $paystart_month && $paystart_year;
 
-      my $payissue       = exists($options{'payissue'})
-                             ? $options{'payissue'}
-                             : $self->payissue;
+      my $payissue       = $options{'payissue'};
       $content{issue_number} = $payissue if $payissue;
 
     } elsif ( $options{method} eq 'ECHECK' ){
-
-      #nop for checks (though it shouldn't be called...)
-
+      #cannot verify, move along (though it shouldn't be called...)
+      return '';
     } else {
-      die "unknown method ". $options{method};
+      return "unknown method ". $options{method};
     }
-
   } elsif ( $namespace eq 'Business::OnlineThirdPartyPayment' ) {
-    #move along
+    #cannot verify, move along
+    return '';
   } else {
-    die "unknown namespace $namespace";
+    return "unknown namespace $namespace";
   }
 
   ###
@@ -1847,6 +1841,7 @@ sub realtime_verify_bop {
 
   my $error;
   my $transaction; #need this back so we can do _tokenize_card
+
   # don't mutex the customer here, because they might be uncommitted. and
   # this is only verification. it doesn't matter if they have other
   # unfinished verifications.
@@ -1859,12 +1854,10 @@ sub realtime_verify_bop {
     'payinfo'           => $options{payinfo},
     'paymask'           => $options{paymask},
     'paydate'           => $paydate,
-    #'recurring_billing' => $content{recurring_billing},
     'pkgnum'            => $options{'pkgnum'},
     'status'            => 'new',
     'gatewaynum'        => $payment_gateway->gatewaynum || '',
     'session_id'        => $options{session_id} || '',
-    #'jobnum'            => $options{depend_jobnum} || '',
   };
   $cust_pay_pending->payunique( $options{payunique} )
     if defined($options{payunique}) && length($options{payunique});
@@ -1905,12 +1898,9 @@ sub realtime_verify_bop {
       'action'         => 'Authorization Only',
       'description'    => $options{'description'},
       'amount'         => '1.00',
-      #'invoice_number' => $options{'invnum'},
       'customer_id'    => $self->custnum,
       %$bop_content,
       'reference'      => $cust_pay_pending->paypendingnum, #for now
-      'callback_url'   => $payment_gateway->gateway_callback_url,
-      'cancel_url'     => $payment_gateway->gateway_cancel_url,
       'email'          => $email,
       %content, #after
     );
@@ -2123,7 +2113,9 @@ sub realtime_verify_bop {
   # Tokenize
   ###
 
-  $self->_tokenize_card($transaction,$options{'payinfo'},$log);
+  #important that we not pass replace option here,
+  #because cust_payby->replace uses realtime_verify_bop!
+  $self->_tokenize_card($transaction,$options{'cust_payby'},$log);
 
   ###
   # result handling
@@ -2135,6 +2127,144 @@ sub realtime_verify_bop {
 
 }
 
+=item realtime_tokenize [ OPTION => VALUE ... ]
+
+If possible, runs a tokenize transaction.
+In order to be possible, a credit card cust_payby record
+must be passed and a Business::OnlinePayment gateway capable
+of Tokenize transactions must be configured for this user.
+
+Returns the empty string if the authorization was sucessful
+or was not possible (thus allowing this to be safely called with
+non-tokenizable records/gateways, without having to perform separate tests),
+or an error message otherwise.
+
+Option I<cust_payby> should be passed, even if it's not yet been inserted.
+Object will be tokenized if possible, but that change will not be
+updated in database (must be inserted/replaced afterwards.)
+
+=cut
+
+sub realtime_tokenize {
+  my $self = shift;
+
+  local($DEBUG) = $FS::cust_main::DEBUG if $FS::cust_main::DEBUG > $DEBUG;
+  my $log = FS::Log->new('FS::cust_main::Billing_Realtime::realtime_tokenize');
+
+  my %options = ();
+  if (ref($_[0]) eq 'HASH') {
+    %options = %{$_[0]};
+  } else {
+    %options = @_;
+  }
+
+  # set fields from passed cust_payby
+  return "No cust_payby" unless $options{'cust_payby'};
+  $self->_bop_cust_payby_options(\%options);
+  return '' unless $options{method} eq 'CC';
+  return '' if $options{payinfo} =~ /^99\d{14}$/; #already tokenized
+
+  ###
+  # select a gateway
+  ###
+
+  $options{'nofatal'} = 1;
+  my $payment_gateway =  $self->_payment_gateway( \%options );
+  return '' unless $payment_gateway;
+  my $namespace = $payment_gateway->gateway_namespace;
+  return '' unless $namespace eq 'Business::OnlinePayment';
+
+  eval "use $namespace";  
+  return $@ if $@;
+
+  ###
+  # check for tokenize ability
+  ###
+
+  # just create transaction now, so it loads gateway_module
+  my $transaction = new $namespace( $payment_gateway->gateway_module,
+                                    $self->_bop_options(\%options),
+                                  );
+
+  my %supported_actions = $transaction->info('supported_actions');
+  return '' unless $supported_actions{'CC'} and grep(/^Tokenize$/,@{$supported_actions{'CC'}});
+
+  ###
+  # check for banned credit card/ACH
+  ###
+
+  my $ban = FS::banned_pay->ban_search(
+    'payby'   => $bop_method2payby{'CC'},
+    'payinfo' => $options{payinfo},
+  );
+  return "Banned credit card" if $ban && $ban->bantype ne 'warn';
+
+  ###
+  # massage data
+  ###
+
+  my $bop_content = $self->_bop_content(\%options);
+  return $bop_content unless ref($bop_content);
+
+  my $paydate = '';
+  my %content = ();
+
+  $content{card_number} = $options{payinfo};
+  $paydate = $options{'paydate'};
+  $paydate =~ /^\d{2}(\d{2})[\/\-](\d+)[\/\-]\d+$/;
+  $content{expiration} = "$2/$1";
+
+  $content{cvv2} = $options{'paycvv'}
+    if length($options{'paycvv'});
+
+  my $paystart_month = $options{'paystart_month'};
+  my $paystart_year  = $options{'paystart_year'};
+
+  $content{card_start} = "$paystart_month/$paystart_year"
+    if $paystart_month && $paystart_year;
+
+  my $payissue       = $options{'payissue'};
+  $content{issue_number} = $payissue if $payissue;
+
+  ###
+  # run transaction
+  ###
+
+  my $error;
+
+  # no cust_pay_pending---this is not a financial transaction
+
+  $transaction->content(
+    'type'           => 'CC',
+    $self->_bop_auth(\%options),          
+    'action'         => 'Tokenize',
+    'description'    => $options{'description'},
+    'customer_id'    => $self->custnum,
+    %$bop_content,
+    %content, #after
+  );
+
+  # no $BOP_TESTING handling for this
+  $transaction->test_transaction(1)
+    if $conf->exists('business-onlinepayment-test_transaction');
+  $transaction->submit();
+
+  if ( $transaction->card_token() ) { # no is_success flag
+
+    #important that we not pass replace option here, 
+    #because cust_payby->replace uses realtime_tokenize!
+    $self->_tokenize_card($transaction,$options{'cust_payby'},$log);
+
+  } else {
+
+    $error = $transaction->error_message || 'Unknown error';
+
+  }
+
+  return $error;
+
+}
+
 =back
 
 =head1 BUGS
diff --git a/FS/FS/cust_payby.pm b/FS/FS/cust_payby.pm
index e4a1d19..626fc9f 100644
--- a/FS/FS/cust_payby.pm
+++ b/FS/FS/cust_payby.pm
@@ -250,8 +250,11 @@ sub replace {
 
     if ( $conf->exists('business-onlinepayment-verification') ) {
       $error = $self->verify;
-      return $error if $error;
+    } else {
+      $error = $self->tokenize;
     }
+    return $error if $error;
+
   }
 
   local $SIG{HUP} = 'IGNORE';
@@ -521,9 +524,12 @@ sub check {
 
   }
 
-  if ( ! $self->custpaybynum
-       && $conf->exists('business-onlinepayment-verification') ) {
-    $error = $self->verify;
+  if ( ! $self->custpaybynum ) {
+    if ($conf->exists('business-onlinepayment-verification')) {
+      $error = $self->verify;
+    } else {
+      $error = $self->tokenize;
+    }
     return $error if $error;
   }
 
@@ -638,59 +644,48 @@ sub label {
 
 =item realtime_bop
 
+Runs a L<realtime_bop|FS::cust_main::Billing_Realtime::realtime_bop> transaction on this card
+
 =cut
 
 sub realtime_bop {
   my( $self, %opt ) = @_;
 
-  $opt{$_} = $self->$_() for qw( payinfo payname paydate );
-
-  if ( $self->locationnum ) {
-    my $cust_location = $self->cust_location;
-    $opt{$_} = $cust_location->$_() for qw( address1 address2 city state zip );
-  }
-
   $self->cust_main->realtime_bop({
-    'method' => FS::payby->payby2bop( $self->payby ),
     %opt,
+    'cust_payby' => $self,
   });
 
 }
 
-=item verify 
+=item tokenize
+
+Runs a L<realtime_tokenize|FS::cust_main::Billing_Realtime::realtime_tokenize> transaction on this card
 
 =cut
 
-sub verify {
+sub tokenize {
   my $self = shift;
   return '' unless $self->payby =~ /^(CARD|DCRD)$/;
 
-  my %opt = ();
+  $self->cust_main->realtime_tokenize({
+    'cust_payby' => $self,
+  });
 
-  # false laziness with check
-  my( $m, $y );
-  if ( $self->paydate =~ /^(\d{1,2})[\/\-](\d{2}(\d{2})?)$/ ) {
-    ( $m, $y ) = ( $1, length($2) == 4 ? $2 : "20$2" );
-  } elsif ( $self->paydate =~ /^19(\d{2})[\/\-](\d{1,2})[\/\-]\d+$/ ) {
-    ( $m, $y ) = ( $2, "19$1" );
-  } elsif ( $self->paydate =~ /^(20)?(\d{2})[\/\-](\d{1,2})[\/\-]\d+$/ ) {
-    ( $m, $y ) = ( $3, "20$2" );
-  } else {
-    return "Illegal expiration date: ". $self->paydate;
-  }
-  $m = sprintf('%02d',$m);
-  $opt{paydate} = "$y-$m-01";
+}
 
-  $opt{$_} = $self->$_() for qw( payinfo payname paycvv );
+=item verify 
 
-  if ( $self->locationnum ) {
-    my $cust_location = $self->cust_location;
-    $opt{$_} = $cust_location->$_() for qw( address1 address2 city state zip );
-  }
+Runs a L<realtime_verify_bop|FS::cust_main::Billing_Realtime/realtime_verify_bop> transaction on this card
+
+=cut
+
+sub verify {
+  my $self = shift;
+  return '' unless $self->payby =~ /^(CARD|DCRD)$/;
 
   $self->cust_main->realtime_verify_bop({
-    'method' => FS::payby->payby2bop( $self->payby ),
-    %opt,
+    'cust_payby' => $self,
   });
 
 }
diff --git a/FS/FS/log_context.pm b/FS/FS/log_context.pm
index 1d98ac1..51aa79d 100644
--- a/FS/FS/log_context.pm
+++ b/FS/FS/log_context.pm
@@ -9,6 +9,7 @@ my @contexts = ( qw(
   FS::cust_main::Billing::bill_and_collect
   FS::cust_main::Billing::bill
   FS::cust_main::Billing_Realtime::realtime_bop
+  FS::cust_main::Billing_Realtime::realtime_tokenize
   FS::cust_main::Billing_Realtime::realtime_verify_bop
   FS::pay_batch::import_from_gateway
   FS::part_pkg
diff --git a/FS/FS/payinfo_Mixin.pm b/FS/FS/payinfo_Mixin.pm
index 5f7ce35..3a32ad5 100644
--- a/FS/FS/payinfo_Mixin.pm
+++ b/FS/FS/payinfo_Mixin.pm
@@ -67,8 +67,9 @@ sub payinfo {
   my($self,$payinfo) = @_;
 
   if ( defined($payinfo) ) {
+    $self->paymask($self->mask_payinfo) unless $self->payinfo =~ /^99\d{14}$/; #make sure old mask is set
     $self->setfield('payinfo', $payinfo);
-    $self->paymask($self->mask_payinfo) unless $payinfo =~ /^99\d{14}$/; #token
+    $self->paymask($self->mask_payinfo) unless $payinfo =~ /^99\d{14}$/; #remask unless tokenizing
   } else {
     $self->getfield('payinfo');
   }
diff --git a/httemplate/misc/process/payment.cgi b/httemplate/misc/process/payment.cgi
index 852becb..74ca734 100644
--- a/httemplate/misc/process/payment.cgi
+++ b/httemplate/misc/process/payment.cgi
@@ -72,7 +72,7 @@ $cgi->param('discount_term') =~ /^(\d*)$/
   or errorpage("illegal discount_term");
 my $discount_term = $1;
 
-my( $payinfo, $paycvv, $month, $year, $payname );
+my( $cust_payby, $payinfo, $paycvv, $month, $year, $payname );
 my $paymask = '';
 if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
 
@@ -80,10 +80,11 @@ if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
   # use stored cust_payby info
   ##
 
-  my $cust_payby = qsearchs('cust_payby', { custnum      => $custnum,
+  $cust_payby = qsearchs('cust_payby', { custnum      => $custnum,
                                             custpaybynum => $custpaybynum, } )
     or die "unknown custpaybynum $custpaybynum";
 
+  # not needed for realtime_bop, but still needed for batch_card
   $payinfo = $cust_payby->payinfo;
   $paymask = $cust_payby->paymask;
   $paycvv = $cust_payby->paycvv; # pass it if we got it, running a transaction will clear it
@@ -164,7 +165,7 @@ if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
     die "unknown payby $payby";
   }
 
-  # save first, for proper tokenization later
+  # save first, for proper tokenization
   if ( $cgi->param('save') ) {
 
     my %saveopt;
@@ -181,6 +182,7 @@ if ( (my $custpaybynum = scalar($cgi->param('custpaybynum'))) > 0 ) {
     }
 
     my $error = $cust_main->save_cust_payby(
+      'saved_cust_payby' => \$cust_payby,
       'payment_payby' => $payby,
       'auto'          => scalar($cgi->param('auto')),
       'weight'        => scalar($cgi->param('weight')),
@@ -220,6 +222,7 @@ if ( $cgi->param('batch') ) {
 } else {
 
   $error = $cust_main->realtime_bop( $FS::payby::payby2bop{$payby}, $amount,
+    'cust_payby' => $cust_payby, # if defined, will override passed payinfo, etc 
     'quiet'      => 1,
     'manual'     => 1,
     'balance'    => $balance,

commit a77a43c3e472c12a2a343d92fd96611a00a704b6
Author: Jonathan Prykop <jonathan at freeside.biz>
Date:   Fri Jul 29 20:35:16 2016 -0500

    RT#71513: Card tokenization in v4+

diff --git a/FS/FS/cust_main/Billing_Realtime.pm b/FS/FS/cust_main/Billing_Realtime.pm
index c49e150..cb7299b 100644
--- a/FS/FS/cust_main/Billing_Realtime.pm
+++ b/FS/FS/cust_main/Billing_Realtime.pm
@@ -355,6 +355,35 @@ sub _bop_content {
   \%content;
 }
 
+sub _tokenize_card {
+  my ($self,$transaction,$payinfo,$log) = @_;
+
+  if ( $transaction->can('card_token') 
+       and $transaction->card_token 
+       and $payinfo !~ /^99\d{14}$/ #not already tokenized
+  ) {
+
+    my @cust_payby = $self->cust_payby('CARD','DCRD');
+    @cust_payby = grep { $payinfo == $_->payinfo } @cust_payby;
+    if (@cust_payby > 1) {
+      $log->error('Multiple matching card numbers for cust '.$self->custnum.', could not tokenize card');
+    } elsif (@cust_payby) {
+      my $cust_payby = $cust_payby[0];
+      $cust_payby->payinfo($transaction->card_token);
+      my $error = $cust_payby->replace;
+      if ( $error ) {
+        $log->error('Error storing token for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum.': '.$error);
+      } else {
+        $log->debug('Tokenized card for cust '.$self->custnum.', cust_payby '.$cust_payby->custpaybynum);
+      }
+    } else {
+      $log->debug('No matching card numbers for cust '.$self->custnum.', could not tokenize card');
+    }
+
+  }
+
+}
+
 my %bop_method2payby = (
   'CC'     => 'CARD',
   'ECHECK' => 'CHEK',
@@ -369,6 +398,8 @@ sub realtime_bop {
     unless $FS::UID::AutoCommit;
 
   local($DEBUG) = $FS::cust_main::DEBUG if $FS::cust_main::DEBUG > $DEBUG;
+
+  my $log = FS::Log->new('FS::cust_main::Billing_Realtime::realtime_bop');
  
   my %options = ();
   if (ref($_[0]) eq 'HASH') {
@@ -774,18 +805,7 @@ sub realtime_bop {
   # Tokenize
   ###
 
-
-  if ( $transaction->can('card_token') && $transaction->card_token ) {
-
-    if ( $options{'payinfo'} eq $self->payinfo ) {
-      $self->payinfo($transaction->card_token);
-      my $error = $self->replace;
-      if ( $error ) {
-        warn "WARNING: error storing token: $error, but proceeding anyway\n";
-      }
-    }
-
-  }
+  $self->_tokenize_card($transaction,$options{'payinfo'},$log);
 
   ###
   # result handling
@@ -2103,19 +2123,7 @@ sub realtime_verify_bop {
   # Tokenize
   ###
 
-  if ( $transaction->can('card_token') && $transaction->card_token ) {
-
-    if ( $options{'payinfo'} eq $self->payinfo ) {
-      $self->payinfo($transaction->card_token);
-      my $error = $self->replace;
-      if ( $error ) {
-        my $warning = "WARNING: error storing token: $error, but proceeding anyway\n";
-        $log->warning($warning);
-        warn $warning;
-      }
-    }
-
-  }
+  $self->_tokenize_card($transaction,$options{'payinfo'},$log);
 
   ###
   # result handling
diff --git a/FS/FS/log_context.pm b/FS/FS/log_context.pm
index afd67cc..1d98ac1 100644
--- a/FS/FS/log_context.pm
+++ b/FS/FS/log_context.pm
@@ -5,10 +5,10 @@ use base qw( FS::Record );
 use FS::Record qw( qsearch qsearchs );
 
 my @contexts = ( qw(
-  test
   bill_and_collect
   FS::cust_main::Billing::bill_and_collect
   FS::cust_main::Billing::bill
+  FS::cust_main::Billing_Realtime::realtime_bop
   FS::cust_main::Billing_Realtime::realtime_verify_bop
   FS::pay_batch::import_from_gateway
   FS::part_pkg
@@ -26,6 +26,7 @@ my @contexts = ( qw(
   upgrade_taxable_billpkgnum
   freeside-paymentech-upload
   freeside-paymentech-download
+  test
 ) );
 
 =head1 NAME

-----------------------------------------------------------------------

Summary of changes:
 FS/FS/ClientAPI/MyAccount.pm                       |   22 +-
 FS/FS/ClientAPI/Signup.pm                          |   71 +-
 FS/FS/Conf.pm                                      |    4 +-
 FS/FS/Cron/cleanup.pm                              |   16 +-
 FS/FS/Cron/tax_rate_update.pm                      |    2 +-
 FS/FS/Upgrade.pm                                   |    8 +
 FS/FS/agent.pm                                     |  145 +--
 FS/FS/agent_payment_gateway.pm                     |   16 +
 FS/FS/banned_pay.pm                                |   10 +-
 FS/FS/cust_main.pm                                 |   14 +-
 FS/FS/cust_main/Billing_Realtime.pm                |  974 ++++++++++++++++----
 FS/FS/cust_payby.pm                                |   71 +-
 FS/FS/cust_refund.pm                               |    3 +-
 FS/FS/log_context.pm                               |    5 +-
 FS/FS/payinfo_Mixin.pm                             |   26 +-
 FS/FS/payinfo_transaction_Mixin.pm                 |    2 -
 FS/FS/payment_gateway.pm                           |  113 +++
 FS/bin/freeside-daily                              |    2 +-
 FS/t/suite/13-tokenization.t                       |  213 +++++
 FS/t/suite/14-tokenization_refund.t                |  246 +++++
 FS/t/suite/15-activate_encryption.t                |   18 +-
 httemplate/browse/agent.cgi                        |   24 +-
 httemplate/edit/agent_payment_gateway.html         |   28 +-
 httemplate/edit/elements/edit.html                 |    2 +-
 httemplate/edit/payment_gateway.html               |   11 +-
 httemplate/edit/process/agent_payment_gateway.html |   13 +-
 httemplate/misc/process/payment.cgi                |   16 +-
 27 files changed, 1603 insertions(+), 472 deletions(-)
 create mode 100755 FS/t/suite/13-tokenization.t
 create mode 100755 FS/t/suite/14-tokenization_refund.t




More information about the freeside-commits mailing list