From gson@

- don't assume -h is always on for large factors
- fix saved large factors printing when -h

1 files changed, 42 insertions, 31 deletions

diff --git a/factor/factor.c b/factor/factor.c
index 0e2c6e76..0cd323ca 100644
--- a/factor/factor.c
+++ b/factor/factor.c
@@ -1,4 +1,4 @@
-/*	$NetBSD: factor.c,v 1.35 2020/10/07 19:48:29 christos Exp $	*/
+/*	$NetBSD: factor.c,v 1.36 2020/10/11 17:18:34 christos Exp $	*/
 /*
  * Copyright (c) 1989, 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -41,7 +41,7 @@ __COPYRIGHT("@(#) Copyright (c) 1989, 1993\
 __SCCSID("@(#)factor.c	8.4 (Berkeley) 5/4/95");
 #endif
 #ifdef __RCSID
-__RCSID("$NetBSD: factor.c,v 1.35 2020/10/07 19:48:29 christos Exp $");
+__RCSID("$NetBSD: factor.c,v 1.36 2020/10/11 17:18:34 christos Exp $");
 #endif
 #ifdef __FBSDID
 __FBSDID("$FreeBSD: head/usr.bin/factor/factor.c 356666 2020-01-12 20:25:11Z gad $");
@@ -91,7 +91,8 @@ __FBSDID("$FreeBSD: head/usr.bin/factor/factor.c 356666 2020-01-12 20:25:11Z gad
 
 #define	PRIME_CHECKS	5
 
-static void	pollard_pminus1(BIGNUM *, int); /* print factors for big numbers */
+/* print factors for big numbers */
+static void	pollard_pminus1(BIGNUM *, int, int);
 
 #else
 
@@ -121,7 +122,7 @@ static void	BN_print_dec_fp(FILE *, const BIGNUM *);
 static void	convert_str2bn(BIGNUM **, char *);
 static bool	is_hex_str(char *);
 static void	pr_fact(BIGNUM *, int, int);	/* print factors of a value */
-static void	pr_print(BIGNUM *, int);	/* print a prime */
+static void	pr_print(BIGNUM *, int, int);	/* print a prime */
 static void	usage(void) __dead;
 
 static BN_CTX	*ctx;			/* just use a global context */
@@ -243,13 +244,13 @@ pr_fact(BIGNUM *val, int hflag, int xflag)
 				errx(1, "error in BN_sqr()");
 			if (BN_cmp(bnfact, val) > 0 ||
 			    BN_is_prime_ex(val, PRIME_CHECKS, NULL, NULL) == 1)
-				pr_print(val, xflag);
+				pr_print(val, hflag, xflag);
 			else
-				pollard_pminus1(val, xflag);
+				pollard_pminus1(val, hflag, xflag);
 #else
-			pr_print(val, xflag);
+			pr_print(val, hflag, xflag);
 #endif
-			pr_print(NULL, xflag);
+			pr_print(NULL, hflag, xflag);
 			break;
 		}
 
@@ -275,39 +276,49 @@ pr_fact(BIGNUM *val, int hflag, int xflag)
 }
 
 static void
-pr_print(BIGNUM *val, int xflag)
+pr_print(BIGNUM *val, int hflag, int xflag)
 {
 	static BIGNUM *sval;
 	static int ex = 1;
-	if (sval == NULL) {
-		sval = BN_dup(val);
-		return;
-	}
+	BIGNUM *pval;
+
+	if (hflag) {
+		if (sval == NULL) {
+			sval = BN_dup(val);
+			return;
+		}
 
-	if (val != NULL && BN_cmp(val, sval) == 0) {
-		ex++;
+		if (val != NULL && BN_cmp(val, sval) == 0) {
+			ex++;
+			return;
+		}
+		pval = sval;
+	} else if (val == NULL) {
 		return;
+	} else {
+		pval = val;
 	}
-	if (val == NULL)
-		val = sval;
 
 	if (xflag) {
 		fputs(" 0x", stdout);
-		BN_print_fp(stdout, val);
+		BN_print_fp(stdout, pval);
 	} else {
 		putchar(' ');
-		BN_print_dec_fp(stdout, val);
+		BN_print_dec_fp(stdout, pval);
 	}
-	if (ex > 1)
-		pr_exp(ex, xflag);
 
-	if (val != NULL) {
-		BN_copy(sval, val);
-	} else {
-		BN_free(sval);
-		sval = NULL;
+	if (hflag) {
+		if (ex > 1)
+			pr_exp(ex, xflag);
+
+		if (val != NULL) {
+			BN_copy(sval, val);
+		} else {
+			BN_free(sval);
+			sval = NULL;
+		}
+		ex = 1;
 	}
-	ex = 1;
 }
 
 static void
@@ -321,7 +332,7 @@ usage(void)
 
 /* pollard p-1, algorithm from Jim Gillogly, May 2000 */
 static void
-pollard_pminus1(BIGNUM *val, int xflag)
+pollard_pminus1(BIGNUM *val, int hflag, int xflag)
 {
 	BIGNUM *base, *rbase, *num, *i, *x;
 
@@ -350,9 +361,9 @@ newbase:
 
 		if (!BN_is_one(x)) {
 			if (BN_is_prime_ex(x, PRIME_CHECKS, NULL, NULL) == 1)
-				pr_print(x, xflag);
+				pr_print(x, hflag, xflag);
 			else
-				pollard_pminus1(x, xflag);
+				pollard_pminus1(x, hflag, xflag);
 			fflush(stdout);
 
 			BN_div(num, NULL, val, x, ctx);
@@ -360,7 +371,7 @@ newbase:
 				return;
 			if (BN_is_prime_ex(num, PRIME_CHECKS, NULL,
 			    NULL) == 1) {
-				pr_print(num, xflag);
+				pr_print(num, hflag, xflag);
 				fflush(stdout);
 				return;
 			}