drivers/misc/sgi-xp/xpc_main.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (c) 2004-2008 Silicon Graphics, Inc.  All Rights Reserved.
   7  */
   8
   9 /*
  10  * Cross Partition Communication (XPC) support - standard version.
  11  *
  12  *      XPC provides a message passing capability that crosses partition
  13  *      boundaries. This module is made up of two parts:
  14  *
  15  *          partition   This part detects the presence/absence of other
  16  *                      partitions. It provides a heartbeat and monitors
  17  *                      the heartbeats of other partitions.
  18  *
  19  *          channel     This part manages the channels and sends/receives
  20  *                      messages across them to/from other partitions.
  21  *
  22  *      There are a couple of additional functions residing in XP, which
  23  *      provide an interface to XPC for its users.
  24  *
  25  *
  26  *      Caveats:
  27  *
  28  *        . Currently on sn2, we have no way to determine which nasid an IRQ
  29  *          came from. Thus, xpc_send_IRQ_sn2() does a remote amo write
  30  *          followed by an IPI. The amo indicates where data is to be pulled
  31  *          from, so after the IPI arrives, the remote partition checks the amo
  32  *          word. The IPI can actually arrive before the amo however, so other
  33  *          code must periodically check for this case. Also, remote amo
  34  *          operations do not reliably time out. Thus we do a remote PIO read
  35  *          solely to know whether the remote partition is down and whether we
  36  *          should stop sending IPIs to it. This remote PIO read operation is
  37  *          set up in a special nofault region so SAL knows to ignore (and
  38  *          cleanup) any errors due to the remote amo write, PIO read, and/or
  39  *          PIO write operations.
  40  *
  41  *          If/when new hardware solves this IPI problem, we should abandon
  42  *          the current approach.
  43  *
  44  */
  45
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/cache.h>
  50 #include <linux/interrupt.h>
  51 #include <linux/delay.h>
  52 #include <linux/reboot.h>
  53 #include <linux/completion.h>
  54 #include <linux/kdebug.h>
  55 #include <linux/kthread.h>
  56 #include <linux/uaccess.h>
  57 #include <asm/sn/intr.h>
  58 #include <asm/sn/sn_sal.h>
  59 #include "xpc.h"
  60
  61 /* define two XPC debug device structures to be used with dev_dbg() et al */
  62
  63 struct device_driver xpc_dbg_name = {
  64         .name = "xpc"
  65 };
  66
  67 struct device xpc_part_dbg_subname = {
  68         .bus_id = {0},          /* set to "part" at xpc_init() time */
  69         .driver = &xpc_dbg_name
  70 };
  71
  72 struct device xpc_chan_dbg_subname = {
  73         .bus_id = {0},          /* set to "chan" at xpc_init() time */
  74         .driver = &xpc_dbg_name
  75 };
  76
  77 struct device *xpc_part = &xpc_part_dbg_subname;
  78 struct device *xpc_chan = &xpc_chan_dbg_subname;
  79
  80 static int xpc_kdebug_ignore;
  81
  82 /* systune related variables for /proc/sys directories */
  83
  84 static int xpc_hb_interval = XPC_HB_DEFAULT_INTERVAL;
  85 static int xpc_hb_min_interval = 1;
  86 static int xpc_hb_max_interval = 10;
  87
  88 static int xpc_hb_check_interval = XPC_HB_CHECK_DEFAULT_INTERVAL;
  89 static int xpc_hb_check_min_interval = 10;
  90 static int xpc_hb_check_max_interval = 120;
  91
  92 int xpc_disengage_timelimit = XPC_DISENGAGE_DEFAULT_TIMELIMIT;
  93 static int xpc_disengage_min_timelimit; /* = 0 */
  94 static int xpc_disengage_max_timelimit = 120;
  95
  96 static ctl_table xpc_sys_xpc_hb_dir[] = {
  97         {
  98          .ctl_name = CTL_UNNUMBERED,
  99          .procname = "hb_interval",
 100          .data = &xpc_hb_interval,
 101          .maxlen = sizeof(int),
 102          .mode = 0644,
 103          .proc_handler = &proc_dointvec_minmax,
 104          .strategy = &sysctl_intvec,
 105          .extra1 = &xpc_hb_min_interval,
 106          .extra2 = &xpc_hb_max_interval},
 107         {
 108          .ctl_name = CTL_UNNUMBERED,
 109          .procname = "hb_check_interval",
 110          .data = &xpc_hb_check_interval,
 111          .maxlen = sizeof(int),
 112          .mode = 0644,
 113          .proc_handler = &proc_dointvec_minmax,
 114          .strategy = &sysctl_intvec,
 115          .extra1 = &xpc_hb_check_min_interval,
 116          .extra2 = &xpc_hb_check_max_interval},
 117         {}
 118 };
 119 static ctl_table xpc_sys_xpc_dir[] = {
 120         {
 121          .ctl_name = CTL_UNNUMBERED,
 122          .procname = "hb",
 123          .mode = 0555,
 124          .child = xpc_sys_xpc_hb_dir},
 125         {
 126          .ctl_name = CTL_UNNUMBERED,
 127          .procname = "disengage_timelimit",
 128          .data = &xpc_disengage_timelimit,
 129          .maxlen = sizeof(int),
 130          .mode = 0644,
 131          .proc_handler = &proc_dointvec_minmax,
 132          .strategy = &sysctl_intvec,
 133          .extra1 = &xpc_disengage_min_timelimit,
 134          .extra2 = &xpc_disengage_max_timelimit},
 135         {}
 136 };
 137 static ctl_table xpc_sys_dir[] = {
 138         {
 139          .ctl_name = CTL_UNNUMBERED,
 140          .procname = "xpc",
 141          .mode = 0555,
 142          .child = xpc_sys_xpc_dir},
 143         {}
 144 };
 145 static struct ctl_table_header *xpc_sysctl;
 146
 147 /* non-zero if any remote partition disengage was timed out */
 148 int xpc_disengage_timedout;
 149
 150 /* #of activate IRQs received */
 151 atomic_t xpc_activate_IRQ_rcvd = ATOMIC_INIT(0);
 152
 153 /* IRQ handler notifies this wait queue on receipt of an IRQ */
 154 DECLARE_WAIT_QUEUE_HEAD(xpc_activate_IRQ_wq);
 155
 156 static unsigned long xpc_hb_check_timeout;
 157 static struct timer_list xpc_hb_timer;
 158 void *xpc_heartbeating_to_mask;
 159
 160 /* notification that the xpc_hb_checker thread has exited */
 161 static DECLARE_COMPLETION(xpc_hb_checker_exited);
 162
 163 /* notification that the xpc_discovery thread has exited */
 164 static DECLARE_COMPLETION(xpc_discovery_exited);
 165
 166 static void xpc_kthread_waitmsgs(struct xpc_partition *, struct xpc_channel *);
 167
 168 static int xpc_system_reboot(struct notifier_block *, unsigned long, void *);
 169 static struct notifier_block xpc_reboot_notifier = {
 170         .notifier_call = xpc_system_reboot,
 171 };
 172
 173 static int xpc_system_die(struct notifier_block *, unsigned long, void *);
 174 static struct notifier_block xpc_die_notifier = {
 175         .notifier_call = xpc_system_die,
 176 };
 177
 178 enum xp_retval (*xpc_rsvd_page_init) (struct xpc_rsvd_page *rp);
 179 void (*xpc_heartbeat_init) (void);
 180 void (*xpc_heartbeat_exit) (void);
 181 void (*xpc_increment_heartbeat) (void);
 182 void (*xpc_offline_heartbeat) (void);
 183 void (*xpc_online_heartbeat) (void);
 184 void (*xpc_check_remote_hb) (void);
 185
 186 enum xp_retval (*xpc_make_first_contact) (struct xpc_partition *part);
 187 void (*xpc_notify_senders_of_disconnect) (struct xpc_channel *ch);
 188 u64 (*xpc_get_chctl_all_flags) (struct xpc_partition *part);
 189 enum xp_retval (*xpc_allocate_msgqueues) (struct xpc_channel *ch);
 190 void (*xpc_free_msgqueues) (struct xpc_channel *ch);
 191 void (*xpc_process_msg_chctl_flags) (struct xpc_partition *part, int ch_number);
 192 int (*xpc_n_of_deliverable_msgs) (struct xpc_channel *ch);
 193 struct xpc_msg *(*xpc_get_deliverable_msg) (struct xpc_channel *ch);
 194
 195 void (*xpc_request_partition_activation) (struct xpc_rsvd_page *remote_rp,
 196                                           u64 remote_rp_pa, int nasid);
 197 void (*xpc_request_partition_reactivation) (struct xpc_partition *part);
 198 void (*xpc_request_partition_deactivation) (struct xpc_partition *part);
 199 void (*xpc_cancel_partition_deactivation_request) (struct xpc_partition *part);
 200
 201 void (*xpc_process_activate_IRQ_rcvd) (int n_IRQs_expected);
 202 enum xp_retval (*xpc_setup_infrastructure) (struct xpc_partition *part);
 203 void (*xpc_teardown_infrastructure) (struct xpc_partition *part);
 204
 205 void (*xpc_indicate_partition_engaged) (struct xpc_partition *part);
 206 int (*xpc_partition_engaged) (short partid);
 207 int (*xpc_any_partition_engaged) (void);
 208 void (*xpc_indicate_partition_disengaged) (struct xpc_partition *part);
 209 void (*xpc_assume_partition_disengaged) (short partid);
 210
 211 void (*xpc_send_chctl_closerequest) (struct xpc_channel *ch,
 212                                      unsigned long *irq_flags);
 213 void (*xpc_send_chctl_closereply) (struct xpc_channel *ch,
 214                                    unsigned long *irq_flags);
 215 void (*xpc_send_chctl_openrequest) (struct xpc_channel *ch,
 216                                     unsigned long *irq_flags);
 217 void (*xpc_send_chctl_openreply) (struct xpc_channel *ch,
 218                                   unsigned long *irq_flags);
 219
 220 enum xp_retval (*xpc_send_msg) (struct xpc_channel *ch, u32 flags,
 221                                 void *payload, u16 payload_size, u8 notify_type,
 222                                 xpc_notify_func func, void *key);
 223 void (*xpc_received_msg) (struct xpc_channel *ch, struct xpc_msg *msg);
 224
 225 /*
 226  * Timer function to enforce the timelimit on the partition disengage.
 227  */
 228 static void
 229 xpc_timeout_partition_disengage(unsigned long data)
 230 {
 231         struct xpc_partition *part = (struct xpc_partition *)data;
 232
 233         DBUG_ON(time_is_after_jiffies(part->disengage_timeout));
 234
 235         (void)xpc_partition_disengaged(part);
 236
 237         DBUG_ON(part->disengage_timeout != 0);
 238         DBUG_ON(xpc_partition_engaged(XPC_PARTID(part)));
 239 }
 240
 241 /*
 242  * Timer to produce the heartbeat.  The timer structures function is
 243  * already set when this is initially called.  A tunable is used to
 244  * specify when the next timeout should occur.
 245  */
 246 static void
 247 xpc_hb_beater(unsigned long dummy)
 248 {
 249         xpc_increment_heartbeat();
 250
 251         if (time_is_before_eq_jiffies(xpc_hb_check_timeout))
 252                 wake_up_interruptible(&xpc_activate_IRQ_wq);
 253
 254         xpc_hb_timer.expires = jiffies + (xpc_hb_interval * HZ);
 255         add_timer(&xpc_hb_timer);
 256 }
 257
 258 static void
 259 xpc_start_hb_beater(void)
 260 {
 261         xpc_heartbeat_init();
 262         init_timer(&xpc_hb_timer);
 263         xpc_hb_timer.function = xpc_hb_beater;
 264         xpc_hb_beater(0);
 265 }
 266
 267 static void
 268 xpc_stop_hb_beater(void)
 269 {
 270         del_timer_sync(&xpc_hb_timer);
 271         xpc_heartbeat_exit();
 272 }
 273
 274 /*
 275  * This thread is responsible for nearly all of the partition
 276  * activation/deactivation.
 277  */
 278 static int
 279 xpc_hb_checker(void *ignore)
 280 {
 281         int last_IRQ_count = 0;
 282         int new_IRQ_count;
 283         int force_IRQ = 0;
 284
 285         /* this thread was marked active by xpc_hb_init() */
 286
 287         set_cpus_allowed_ptr(current, &cpumask_of_cpu(XPC_HB_CHECK_CPU));
 288
 289         /* set our heartbeating to other partitions into motion */
 290         xpc_hb_check_timeout = jiffies + (xpc_hb_check_interval * HZ);
 291         xpc_start_hb_beater();
 292
 293         while (!xpc_exiting) {
 294
 295                 dev_dbg(xpc_part, "woke up with %d ticks rem; %d IRQs have "
 296                         "been received\n",
 297                         (int)(xpc_hb_check_timeout - jiffies),
 298                         atomic_read(&xpc_activate_IRQ_rcvd) - last_IRQ_count);
 299
 300                 /* checking of remote heartbeats is skewed by IRQ handling */
 301                 if (time_is_before_eq_jiffies(xpc_hb_check_timeout)) {
 302                         dev_dbg(xpc_part, "checking remote heartbeats\n");
 303                         xpc_check_remote_hb();
 304
 305                         /*
 306                          * We need to periodically recheck to ensure no
 307                          * IRQ/amo pairs have been missed.  That check
 308                          * must always reset xpc_hb_check_timeout.
 309                          */
 310                         force_IRQ = 1;
 311                 }
 312
 313                 /* check for outstanding IRQs */
 314                 new_IRQ_count = atomic_read(&xpc_activate_IRQ_rcvd);
 315                 if (last_IRQ_count < new_IRQ_count || force_IRQ != 0) {
 316                         force_IRQ = 0;
 317
 318                         dev_dbg(xpc_part, "found an IRQ to process; will be "
 319                                 "resetting xpc_hb_check_timeout\n");
 320
 321                         xpc_process_activate_IRQ_rcvd(new_IRQ_count -
 322                                                       last_IRQ_count);
 323                         last_IRQ_count = new_IRQ_count;
 324
 325                         xpc_hb_check_timeout = jiffies +
 326                             (xpc_hb_check_interval * HZ);
 327                 }
 328
 329                 /* wait for IRQ or timeout */
 330                 (void)wait_event_interruptible(xpc_activate_IRQ_wq,
 331                                                (last_IRQ_count < atomic_read(
 332                                                 &xpc_activate_IRQ_rcvd)
 333                                                 || time_is_before_eq_jiffies(
 334                                                 xpc_hb_check_timeout) ||
 335                                                 xpc_exiting));
 336         }
 337
 338         xpc_stop_hb_beater();
 339
 340         dev_dbg(xpc_part, "heartbeat checker is exiting\n");
 341
 342         /* mark this thread as having exited */
 343         complete(&xpc_hb_checker_exited);
 344         return 0;
 345 }
 346
 347 /*
 348  * This thread will attempt to discover other partitions to activate
 349  * based on info provided by SAL. This new thread is short lived and
 350  * will exit once discovery is complete.
 351  */
 352 static int
 353 xpc_initiate_discovery(void *ignore)
 354 {
 355         xpc_discovery();
 356
 357         dev_dbg(xpc_part, "discovery thread is exiting\n");
 358
 359         /* mark this thread as having exited */
 360         complete(&xpc_discovery_exited);
 361         return 0;
 362 }
 363
 364 /*
 365  * The first kthread assigned to a newly activated partition is the one
 366  * created by XPC HB with which it calls xpc_activating(). XPC hangs on to
 367  * that kthread until the partition is brought down, at which time that kthread
 368  * returns back to XPC HB. (The return of that kthread will signify to XPC HB
 369  * that XPC has dismantled all communication infrastructure for the associated
 370  * partition.) This kthread becomes the channel manager for that partition.
 371  *
 372  * Each active partition has a channel manager, who, besides connecting and
 373  * disconnecting channels, will ensure that each of the partition's connected
 374  * channels has the required number of assigned kthreads to get the work done.
 375  */
 376 static void
 377 xpc_channel_mgr(struct xpc_partition *part)
 378 {
 379         while (part->act_state != XPC_P_DEACTIVATING ||
 380                atomic_read(&part->nchannels_active) > 0 ||
 381                !xpc_partition_disengaged(part)) {
 382
 383                 xpc_process_sent_chctl_flags(part);
 384
 385                 /*
 386                  * Wait until we've been requested to activate kthreads or
 387                  * all of the channel's message queues have been torn down or
 388                  * a signal is pending.
 389                  *
 390                  * The channel_mgr_requests is set to 1 after being awakened,
 391                  * This is done to prevent the channel mgr from making one pass
 392                  * through the loop for each request, since he will
 393                  * be servicing all the requests in one pass. The reason it's
 394                  * set to 1 instead of 0 is so that other kthreads will know
 395                  * that the channel mgr is running and won't bother trying to
 396                  * wake him up.
 397                  */
 398                 atomic_dec(&part->channel_mgr_requests);
 399                 (void)wait_event_interruptible(part->channel_mgr_wq,
 400                                 (atomic_read(&part->channel_mgr_requests) > 0 ||
 401                                  part->chctl.all_flags != 0 ||
 402                                  (part->act_state == XPC_P_DEACTIVATING &&
 403                                  atomic_read(&part->nchannels_active) == 0 &&
 404                                  xpc_partition_disengaged(part))));
 405                 atomic_set(&part->channel_mgr_requests, 1);
 406         }
 407 }
 408
 409 /*
 410  * When XPC HB determines that a partition has come up, it will create a new
 411  * kthread and that kthread will call this function to attempt to set up the
 412  * basic infrastructure used for Cross Partition Communication with the newly
 413  * upped partition.
 414  *
 415  * The kthread that was created by XPC HB and which setup the XPC
 416  * infrastructure will remain assigned to the partition becoming the channel
 417  * manager for that partition until the partition is deactivating, at which
 418  * time the kthread will teardown the XPC infrastructure and then exit.
 419  */
 420 static int
 421 xpc_activating(void *__partid)
 422 {
 423         short partid = (u64)__partid;
 424         struct xpc_partition *part = &xpc_partitions[partid];
 425         unsigned long irq_flags;
 426
 427         DBUG_ON(partid < 0 || partid >= xp_max_npartitions);
 428
 429         spin_lock_irqsave(&part->act_lock, irq_flags);
 430
 431         if (part->act_state == XPC_P_DEACTIVATING) {
 432                 part->act_state = XPC_P_INACTIVE;
 433                 spin_unlock_irqrestore(&part->act_lock, irq_flags);
 434                 part->remote_rp_pa = 0;
 435                 return 0;
 436         }
 437
 438         /* indicate the thread is activating */
 439         DBUG_ON(part->act_state != XPC_P_ACTIVATION_REQ);
 440         part->act_state = XPC_P_ACTIVATING;
 441
 442         XPC_SET_REASON(part, 0, 0);
 443         spin_unlock_irqrestore(&part->act_lock, irq_flags);
 444
 445         dev_dbg(xpc_part, "activating partition %d\n", partid);
 446
 447         xpc_allow_hb(partid);
 448
 449         if (xpc_setup_infrastructure(part) == xpSuccess) {
 450                 (void)xpc_part_ref(part);       /* this will always succeed */
 451
 452                 if (xpc_make_first_contact(part) == xpSuccess) {
 453                         xpc_mark_partition_active(part);
 454                         xpc_channel_mgr(part);
 455                         /* won't return until partition is deactivating */
 456                 }
 457
 458                 xpc_part_deref(part);
 459                 xpc_teardown_infrastructure(part);
 460         }
 461
 462         xpc_disallow_hb(partid);
 463         xpc_mark_partition_inactive(part);
 464
 465         if (part->reason == xpReactivating) {
 466                 /* interrupting ourselves results in activating partition */
 467                 xpc_request_partition_reactivation(part);
 468         }
 469
 470         return 0;
 471 }
 472
 473 void
 474 xpc_activate_partition(struct xpc_partition *part)
 475 {
 476         short partid = XPC_PARTID(part);
 477         unsigned long irq_flags;
 478         struct task_struct *kthread;
 479
 480         spin_lock_irqsave(&part->act_lock, irq_flags);
 481
 482         DBUG_ON(part->act_state != XPC_P_INACTIVE);
 483
 484         part->act_state = XPC_P_ACTIVATION_REQ;
 485         XPC_SET_REASON(part, xpCloneKThread, __LINE__);
 486
 487         spin_unlock_irqrestore(&part->act_lock, irq_flags);
 488
 489         kthread = kthread_run(xpc_activating, (void *)((u64)partid), "xpc%02d",
 490                               partid);
 491         if (IS_ERR(kthread)) {
 492                 spin_lock_irqsave(&part->act_lock, irq_flags);
 493                 part->act_state = XPC_P_INACTIVE;
 494                 XPC_SET_REASON(part, xpCloneKThreadFailed, __LINE__);
 495                 spin_unlock_irqrestore(&part->act_lock, irq_flags);
 496         }
 497 }
 498
 499 void
 500 xpc_activate_kthreads(struct xpc_channel *ch, int needed)
 501 {
 502         int idle = atomic_read(&ch->kthreads_idle);
 503         int assigned = atomic_read(&ch->kthreads_assigned);
 504         int wakeup;
 505
 506         DBUG_ON(needed <= 0);
 507
 508         if (idle > 0) {
 509                 wakeup = (needed > idle) ? idle : needed;
 510                 needed -= wakeup;
 511
 512                 dev_dbg(xpc_chan, "wakeup %d idle kthreads, partid=%d, "
 513                         "channel=%d\n", wakeup, ch->partid, ch->number);
 514
 515                 /* only wakeup the requested number of kthreads */
 516                 wake_up_nr(&ch->idle_wq, wakeup);
 517         }
 518
 519         if (needed <= 0)
 520                 return;
 521
 522         if (needed + assigned > ch->kthreads_assigned_limit) {
 523                 needed = ch->kthreads_assigned_limit - assigned;
 524                 if (needed <= 0)
 525                         return;
 526         }
 527
 528         dev_dbg(xpc_chan, "create %d new kthreads, partid=%d, channel=%d\n",
 529                 needed, ch->partid, ch->number);
 530
 531         xpc_create_kthreads(ch, needed, 0);
 532 }
 533
 534 /*
 535  * This function is where XPC's kthreads wait for messages to deliver.
 536  */
 537 static void
 538 xpc_kthread_waitmsgs(struct xpc_partition *part, struct xpc_channel *ch)
 539 {
 540         do {
 541                 /* deliver messages to their intended recipients */
 542
 543                 while (xpc_n_of_deliverable_msgs(ch) > 0 &&
 544                        !(ch->flags & XPC_C_DISCONNECTING)) {
 545                         xpc_deliver_msg(ch);
 546                 }
 547
 548                 if (atomic_inc_return(&ch->kthreads_idle) >
 549                     ch->kthreads_idle_limit) {
 550                         /* too many idle kthreads on this channel */
 551                         atomic_dec(&ch->kthreads_idle);
 552                         break;
 553                 }
 554
 555                 dev_dbg(xpc_chan, "idle kthread calling "
 556                         "wait_event_interruptible_exclusive()\n");
 557
 558                 (void)wait_event_interruptible_exclusive(ch->idle_wq,
 559                                 (xpc_n_of_deliverable_msgs(ch) > 0 ||
 560                                  (ch->flags & XPC_C_DISCONNECTING)));
 561
 562                 atomic_dec(&ch->kthreads_idle);
 563
 564         } while (!(ch->flags & XPC_C_DISCONNECTING));
 565 }
 566
 567 static int
 568 xpc_kthread_start(void *args)
 569 {
 570         short partid = XPC_UNPACK_ARG1(args);
 571         u16 ch_number = XPC_UNPACK_ARG2(args);
 572         struct xpc_partition *part = &xpc_partitions[partid];
 573         struct xpc_channel *ch;
 574         int n_needed;
 575         unsigned long irq_flags;
 576
 577         dev_dbg(xpc_chan, "kthread starting, partid=%d, channel=%d\n",
 578                 partid, ch_number);
 579
 580         ch = &part->channels[ch_number];
 581
 582         if (!(ch->flags & XPC_C_DISCONNECTING)) {
 583
 584                 /* let registerer know that connection has been established */
 585
 586                 spin_lock_irqsave(&ch->lock, irq_flags);
 587                 if (!(ch->flags & XPC_C_CONNECTEDCALLOUT)) {
 588                         ch->flags |= XPC_C_CONNECTEDCALLOUT;
 589                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 590
 591                         xpc_connected_callout(ch);
 592
 593                         spin_lock_irqsave(&ch->lock, irq_flags);
 594                         ch->flags |= XPC_C_CONNECTEDCALLOUT_MADE;
 595                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 596
 597                         /*
 598                          * It is possible that while the callout was being
 599                          * made that the remote partition sent some messages.
 600                          * If that is the case, we may need to activate
 601                          * additional kthreads to help deliver them. We only
 602                          * need one less than total #of messages to deliver.
 603                          */
 604                         n_needed = xpc_n_of_deliverable_msgs(ch) - 1;
 605                         if (n_needed > 0 && !(ch->flags & XPC_C_DISCONNECTING))
 606                                 xpc_activate_kthreads(ch, n_needed);
 607
 608                 } else {
 609                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 610                 }
 611
 612                 xpc_kthread_waitmsgs(part, ch);
 613         }
 614
 615         /* let registerer know that connection is disconnecting */
 616
 617         spin_lock_irqsave(&ch->lock, irq_flags);
 618         if ((ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) &&
 619             !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) {
 620                 ch->flags |= XPC_C_DISCONNECTINGCALLOUT;
 621                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 622
 623                 xpc_disconnect_callout(ch, xpDisconnecting);
 624
 625                 spin_lock_irqsave(&ch->lock, irq_flags);
 626                 ch->flags |= XPC_C_DISCONNECTINGCALLOUT_MADE;
 627         }
 628         spin_unlock_irqrestore(&ch->lock, irq_flags);
 629
 630         if (atomic_dec_return(&ch->kthreads_assigned) == 0 &&
 631             atomic_dec_return(&part->nchannels_engaged) == 0) {
 632                 xpc_indicate_partition_disengaged(part);
 633         }
 634
 635         xpc_msgqueue_deref(ch);
 636
 637         dev_dbg(xpc_chan, "kthread exiting, partid=%d, channel=%d\n",
 638                 partid, ch_number);
 639
 640         xpc_part_deref(part);
 641         return 0;
 642 }
 643
 644 /*
 645  * For each partition that XPC has established communications with, there is
 646  * a minimum of one kernel thread assigned to perform any operation that
 647  * may potentially sleep or block (basically the callouts to the asynchronous
 648  * functions registered via xpc_connect()).
 649  *
 650  * Additional kthreads are created and destroyed by XPC as the workload
 651  * demands.
 652  *
 653  * A kthread is assigned to one of the active channels that exists for a given
 654  * partition.
 655  */
 656 void
 657 xpc_create_kthreads(struct xpc_channel *ch, int needed,
 658                     int ignore_disconnecting)
 659 {
 660         unsigned long irq_flags;
 661         u64 args = XPC_PACK_ARGS(ch->partid, ch->number);
 662         struct xpc_partition *part = &xpc_partitions[ch->partid];
 663         struct task_struct *kthread;
 664
 665         while (needed-- > 0) {
 666
 667                 /*
 668                  * The following is done on behalf of the newly created
 669                  * kthread. That kthread is responsible for doing the
 670                  * counterpart to the following before it exits.
 671                  */
 672                 if (ignore_disconnecting) {
 673                         if (!atomic_inc_not_zero(&ch->kthreads_assigned)) {
 674                                 /* kthreads assigned had gone to zero */
 675                                 BUG_ON(!(ch->flags &
 676                                          XPC_C_DISCONNECTINGCALLOUT_MADE));
 677                                 break;
 678                         }
 679
 680                 } else if (ch->flags & XPC_C_DISCONNECTING) {
 681                         break;
 682
 683                 } else if (atomic_inc_return(&ch->kthreads_assigned) == 1 &&
 684                            atomic_inc_return(&part->nchannels_engaged) == 1) {
 685                                 xpc_indicate_partition_engaged(part);
 686                 }
 687                 (void)xpc_part_ref(part);
 688                 xpc_msgqueue_ref(ch);
 689
 690                 kthread = kthread_run(xpc_kthread_start, (void *)args,
 691                                       "xpc%02dc%d", ch->partid, ch->number);
 692                 if (IS_ERR(kthread)) {
 693                         /* the fork failed */
 694
 695                         /*
 696                          * NOTE: if (ignore_disconnecting &&
 697                          * !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) is true,
 698                          * then we'll deadlock if all other kthreads assigned
 699                          * to this channel are blocked in the channel's
 700                          * registerer, because the only thing that will unblock
 701                          * them is the xpDisconnecting callout that this
 702                          * failed kthread_run() would have made.
 703                          */
 704
 705                         if (atomic_dec_return(&ch->kthreads_assigned) == 0 &&
 706                             atomic_dec_return(&part->nchannels_engaged) == 0) {
 707                                 xpc_indicate_partition_disengaged(part);
 708                         }
 709                         xpc_msgqueue_deref(ch);
 710                         xpc_part_deref(part);
 711
 712                         if (atomic_read(&ch->kthreads_assigned) <
 713                             ch->kthreads_idle_limit) {
 714                                 /*
 715                                  * Flag this as an error only if we have an
 716                                  * insufficient #of kthreads for the channel
 717                                  * to function.
 718                                  */
 719                                 spin_lock_irqsave(&ch->lock, irq_flags);
 720                                 XPC_DISCONNECT_CHANNEL(ch, xpLackOfResources,
 721                                                        &irq_flags);
 722                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 723                         }
 724                         break;
 725                 }
 726         }
 727 }
 728
 729 void
 730 xpc_disconnect_wait(int ch_number)
 731 {
 732         unsigned long irq_flags;
 733         short partid;
 734         struct xpc_partition *part;
 735         struct xpc_channel *ch;
 736         int wakeup_channel_mgr;
 737
 738         /* now wait for all callouts to the caller's function to cease */
 739         for (partid = 0; partid < xp_max_npartitions; partid++) {
 740                 part = &xpc_partitions[partid];
 741
 742                 if (!xpc_part_ref(part))
 743                         continue;
 744
 745                 ch = &part->channels[ch_number];
 746
 747                 if (!(ch->flags & XPC_C_WDISCONNECT)) {
 748                         xpc_part_deref(part);
 749                         continue;
 750                 }
 751
 752                 wait_for_completion(&ch->wdisconnect_wait);
 753
 754                 spin_lock_irqsave(&ch->lock, irq_flags);
 755                 DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED));
 756                 wakeup_channel_mgr = 0;
 757
 758                 if (ch->delayed_chctl_flags) {
 759                         if (part->act_state != XPC_P_DEACTIVATING) {
 760                                 spin_lock(&part->chctl_lock);
 761                                 part->chctl.flags[ch->number] |=
 762                                     ch->delayed_chctl_flags;
 763                                 spin_unlock(&part->chctl_lock);
 764                                 wakeup_channel_mgr = 1;
 765                         }
 766                         ch->delayed_chctl_flags = 0;
 767                 }
 768
 769                 ch->flags &= ~XPC_C_WDISCONNECT;
 770                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 771
 772                 if (wakeup_channel_mgr)
 773                         xpc_wakeup_channel_mgr(part);
 774
 775                 xpc_part_deref(part);
 776         }
 777 }
 778
 779 static void
 780 xpc_do_exit(enum xp_retval reason)
 781 {
 782         short partid;
 783         int active_part_count, printed_waiting_msg = 0;
 784         struct xpc_partition *part;
 785         unsigned long printmsg_time, disengage_timeout = 0;
 786
 787         /* a 'rmmod XPC' and a 'reboot' cannot both end up here together */
 788         DBUG_ON(xpc_exiting == 1);
 789
 790         /*
 791          * Let the heartbeat checker thread and the discovery thread
 792          * (if one is running) know that they should exit. Also wake up
 793          * the heartbeat checker thread in case it's sleeping.
 794          */
 795         xpc_exiting = 1;
 796         wake_up_interruptible(&xpc_activate_IRQ_wq);
 797
 798         /* wait for the discovery thread to exit */
 799         wait_for_completion(&xpc_discovery_exited);
 800
 801         /* wait for the heartbeat checker thread to exit */
 802         wait_for_completion(&xpc_hb_checker_exited);
 803
 804         /* sleep for a 1/3 of a second or so */
 805         (void)msleep_interruptible(300);
 806
 807         /* wait for all partitions to become inactive */
 808
 809         printmsg_time = jiffies + (XPC_DEACTIVATE_PRINTMSG_INTERVAL * HZ);
 810         xpc_disengage_timedout = 0;
 811
 812         do {
 813                 active_part_count = 0;
 814
 815                 for (partid = 0; partid < xp_max_npartitions; partid++) {
 816                         part = &xpc_partitions[partid];
 817
 818                         if (xpc_partition_disengaged(part) &&
 819                             part->act_state == XPC_P_INACTIVE) {
 820                                 continue;
 821                         }
 822
 823                         active_part_count++;
 824
 825                         XPC_DEACTIVATE_PARTITION(part, reason);
 826
 827                         if (part->disengage_timeout > disengage_timeout)
 828                                 disengage_timeout = part->disengage_timeout;
 829                 }
 830
 831                 if (xpc_any_partition_engaged()) {
 832                         if (time_is_before_jiffies(printmsg_time)) {
 833                                 dev_info(xpc_part, "waiting for remote "
 834                                          "partitions to deactivate, timeout in "
 835                                          "%ld seconds\n", (disengage_timeout -
 836                                          jiffies) / HZ);
 837                                 printmsg_time = jiffies +
 838                                     (XPC_DEACTIVATE_PRINTMSG_INTERVAL * HZ);
 839                                 printed_waiting_msg = 1;
 840                         }
 841
 842                 } else if (active_part_count > 0) {
 843                         if (printed_waiting_msg) {
 844                                 dev_info(xpc_part, "waiting for local partition"
 845                                          " to deactivate\n");
 846                                 printed_waiting_msg = 0;
 847                         }
 848
 849                 } else {
 850                         if (!xpc_disengage_timedout) {
 851                                 dev_info(xpc_part, "all partitions have "
 852                                          "deactivated\n");
 853                         }
 854                         break;
 855                 }
 856
 857                 /* sleep for a 1/3 of a second or so */
 858                 (void)msleep_interruptible(300);
 859
 860         } while (1);
 861
 862         DBUG_ON(xpc_any_partition_engaged());
 863         DBUG_ON(xpc_any_hbs_allowed() != 0);
 864
 865         /* indicate to others that our reserved page is uninitialized */
 866         xpc_rsvd_page->stamp = 0;
 867
 868         if (reason == xpUnloading) {
 869                 (void)unregister_die_notifier(&xpc_die_notifier);
 870                 (void)unregister_reboot_notifier(&xpc_reboot_notifier);
 871         }
 872
 873         /* clear the interface to XPC's functions */
 874         xpc_clear_interface();
 875
 876         if (xpc_sysctl)
 877                 unregister_sysctl_table(xpc_sysctl);
 878
 879         kfree(xpc_partitions);
 880
 881         if (is_shub())
 882                 xpc_exit_sn2();
 883         else
 884                 xpc_exit_uv();
 885 }
 886
 887 /*
 888  * This function is called when the system is being rebooted.
 889  */
 890 static int
 891 xpc_system_reboot(struct notifier_block *nb, unsigned long event, void *unused)
 892 {
 893         enum xp_retval reason;
 894
 895         switch (event) {
 896         case SYS_RESTART:
 897                 reason = xpSystemReboot;
 898                 break;
 899         case SYS_HALT:
 900                 reason = xpSystemHalt;
 901                 break;
 902         case SYS_POWER_OFF:
 903                 reason = xpSystemPoweroff;
 904                 break;
 905         default:
 906                 reason = xpSystemGoingDown;
 907         }
 908
 909         xpc_do_exit(reason);
 910         return NOTIFY_DONE;
 911 }
 912
 913 /*
 914  * Notify other partitions to deactivate from us by first disengaging from all
 915  * references to our memory.
 916  */
 917 static void
 918 xpc_die_deactivate(void)
 919 {
 920         struct xpc_partition *part;
 921         short partid;
 922         int any_engaged;
 923         long time, printmsg_time, disengage_timeout;
 924
 925         /* keep xpc_hb_checker thread from doing anything (just in case) */
 926         xpc_exiting = 1;
 927
 928         xpc_disallow_all_hbs(); /*indicate we're deactivated */
 929
 930         for (partid = 0; partid < xp_max_npartitions; partid++) {
 931                 part = &xpc_partitions[partid];
 932
 933                 if (xpc_partition_engaged(partid) ||
 934                     part->act_state != XPC_P_INACTIVE) {
 935                         xpc_request_partition_deactivation(part);
 936                         xpc_indicate_partition_disengaged(part);
 937                 }
 938         }
 939
 940         time = rtc_time();
 941         printmsg_time = time +
 942             (XPC_DEACTIVATE_PRINTMSG_INTERVAL * sn_rtc_cycles_per_second);
 943         disengage_timeout = time +
 944             (xpc_disengage_timelimit * sn_rtc_cycles_per_second);
 945
 946         /*
 947          * Though we requested that all other partitions deactivate from us,
 948          * we only wait until they've all disengaged.
 949          */
 950
 951         while (1) {
 952                 any_engaged = xpc_any_partition_engaged();
 953                 if (!any_engaged) {
 954                         dev_info(xpc_part, "all partitions have deactivated\n");
 955                         break;
 956                 }
 957
 958                 time = rtc_time();
 959                 if (time >= disengage_timeout) {
 960                         for (partid = 0; partid < xp_max_npartitions;
 961                              partid++) {
 962                                 if (xpc_partition_engaged(partid)) {
 963                                         dev_info(xpc_part, "deactivate from "
 964                                                  "remote partition %d timed "
 965                                                  "out\n", partid);
 966                                 }
 967                         }
 968                         break;
 969                 }
 970
 971                 if (time >= printmsg_time) {
 972                         dev_info(xpc_part, "waiting for remote partitions to "
 973                                  "deactivate, timeout in %ld seconds\n",
 974                                  (disengage_timeout - time) /
 975                                  sn_rtc_cycles_per_second);
 976                         printmsg_time = time +
 977                             (XPC_DEACTIVATE_PRINTMSG_INTERVAL *
 978                              sn_rtc_cycles_per_second);
 979                 }
 980         }
 981 }
 982
 983 /*
 984  * This function is called when the system is being restarted or halted due
 985  * to some sort of system failure. If this is the case we need to notify the
 986  * other partitions to disengage from all references to our memory.
 987  * This function can also be called when our heartbeater could be offlined
 988  * for a time. In this case we need to notify other partitions to not worry
 989  * about the lack of a heartbeat.
 990  */
 991 static int
 992 xpc_system_die(struct notifier_block *nb, unsigned long event, void *unused)
 993 {
 994         switch (event) {
 995         case DIE_MACHINE_RESTART:
 996         case DIE_MACHINE_HALT:
 997                 xpc_die_deactivate();
 998                 break;
 999
1000         case DIE_KDEBUG_ENTER:
1001                 /* Should lack of heartbeat be ignored by other partitions? */
1002                 if (!xpc_kdebug_ignore)
1003                         break;
1004
1005                 /* fall through */
1006         case DIE_MCA_MONARCH_ENTER:
1007         case DIE_INIT_MONARCH_ENTER:
1008                 xpc_offline_heartbeat();
1009                 break;
1010
1011         case DIE_KDEBUG_LEAVE:
1012                 /* Is lack of heartbeat being ignored by other partitions? */
1013                 if (!xpc_kdebug_ignore)
1014                         break;
1015
1016                 /* fall through */
1017         case DIE_MCA_MONARCH_LEAVE:
1018         case DIE_INIT_MONARCH_LEAVE:
1019                 xpc_online_heartbeat();
1020                 break;
1021         }
1022
1023         return NOTIFY_DONE;
1024 }
1025
1026 int __init
1027 xpc_init(void)
1028 {
1029         int ret;
1030         short partid;
1031         struct xpc_partition *part;
1032         struct task_struct *kthread;
1033
1034         snprintf(xpc_part->bus_id, BUS_ID_SIZE, "part");
1035         snprintf(xpc_chan->bus_id, BUS_ID_SIZE, "chan");
1036
1037         if (is_shub()) {
1038                 /*
1039                  * The ia64-sn2 architecture supports at most 64 partitions.
1040                  * And the inability to unregister remote amos restricts us
1041                  * further to only support exactly 64 partitions on this
1042                  * architecture, no less.
1043                  */
1044                 if (xp_max_npartitions != 64)
1045                         return -EINVAL;
1046
1047                 ret = xpc_init_sn2();
1048                 if (ret != 0)
1049                         return ret;
1050
1051         } else if (is_uv()) {
1052                 xpc_init_uv();
1053
1054         } else {
1055                 return -ENODEV;
1056         }
1057
1058         xpc_partitions = kzalloc(sizeof(struct xpc_partition) *
1059                                  xp_max_npartitions, GFP_KERNEL);
1060         if (xpc_partitions == NULL) {
1061                 dev_err(xpc_part, "can't get memory for partition structure\n");
1062                 ret = -ENOMEM;
1063                 goto out_1;
1064         }
1065
1066         /*
1067          * The first few fields of each entry of xpc_partitions[] need to
1068          * be initialized now so that calls to xpc_connect() and
1069          * xpc_disconnect() can be made prior to the activation of any remote
1070          * partition. NOTE THAT NONE OF THE OTHER FIELDS BELONGING TO THESE
1071          * ENTRIES ARE MEANINGFUL UNTIL AFTER AN ENTRY'S CORRESPONDING
1072          * PARTITION HAS BEEN ACTIVATED.
1073          */
1074         for (partid = 0; partid < xp_max_npartitions; partid++) {
1075                 part = &xpc_partitions[partid];
1076
1077                 DBUG_ON((u64)part != L1_CACHE_ALIGN((u64)part));
1078
1079                 part->activate_IRQ_rcvd = 0;
1080                 spin_lock_init(&part->act_lock);
1081                 part->act_state = XPC_P_INACTIVE;
1082                 XPC_SET_REASON(part, 0, 0);
1083
1084                 init_timer(&part->disengage_timer);
1085                 part->disengage_timer.function =
1086                     xpc_timeout_partition_disengage;
1087                 part->disengage_timer.data = (unsigned long)part;
1088
1089                 part->setup_state = XPC_P_UNSET;
1090                 init_waitqueue_head(&part->teardown_wq);
1091                 atomic_set(&part->references, 0);
1092         }
1093
1094         xpc_sysctl = register_sysctl_table(xpc_sys_dir);
1095
1096         /*
1097          * Fill the partition reserved page with the information needed by
1098          * other partitions to discover we are alive and establish initial
1099          * communications.
1100          */
1101         xpc_rsvd_page = xpc_setup_rsvd_page();
1102         if (xpc_rsvd_page == NULL) {
1103                 dev_err(xpc_part, "can't setup our reserved page\n");
1104                 ret = -EBUSY;
1105                 goto out_2;
1106         }
1107
1108         /* add ourselves to the reboot_notifier_list */
1109         ret = register_reboot_notifier(&xpc_reboot_notifier);
1110         if (ret != 0)
1111                 dev_warn(xpc_part, "can't register reboot notifier\n");
1112
1113         /* add ourselves to the die_notifier list */
1114         ret = register_die_notifier(&xpc_die_notifier);
1115         if (ret != 0)
1116                 dev_warn(xpc_part, "can't register die notifier\n");
1117
1118         /*
1119          * The real work-horse behind xpc.  This processes incoming
1120          * interrupts and monitors remote heartbeats.
1121          */
1122         kthread = kthread_run(xpc_hb_checker, NULL, XPC_HB_CHECK_THREAD_NAME);
1123         if (IS_ERR(kthread)) {
1124                 dev_err(xpc_part, "failed while forking hb check thread\n");
1125                 ret = -EBUSY;
1126                 goto out_3;
1127         }
1128
1129         /*
1130          * Startup a thread that will attempt to discover other partitions to
1131          * activate based on info provided by SAL. This new thread is short
1132          * lived and will exit once discovery is complete.
1133          */
1134         kthread = kthread_run(xpc_initiate_discovery, NULL,
1135                               XPC_DISCOVERY_THREAD_NAME);
1136         if (IS_ERR(kthread)) {
1137                 dev_err(xpc_part, "failed while forking discovery thread\n");
1138
1139                 /* mark this new thread as a non-starter */
1140                 complete(&xpc_discovery_exited);
1141
1142                 xpc_do_exit(xpUnloading);
1143                 return -EBUSY;
1144         }
1145
1146         /* set the interface to point at XPC's functions */
1147         xpc_set_interface(xpc_initiate_connect, xpc_initiate_disconnect,
1148                           xpc_initiate_send, xpc_initiate_send_notify,
1149                           xpc_initiate_received, xpc_initiate_partid_to_nasids);
1150
1151         return 0;
1152
1153         /* initialization was not successful */
1154 out_3:
1155         /* indicate to others that our reserved page is uninitialized */
1156         xpc_rsvd_page->stamp = 0;
1157
1158         (void)unregister_die_notifier(&xpc_die_notifier);
1159         (void)unregister_reboot_notifier(&xpc_reboot_notifier);
1160 out_2:
1161         if (xpc_sysctl)
1162                 unregister_sysctl_table(xpc_sysctl);
1163         kfree(xpc_partitions);
1164 out_1:
1165         if (is_shub())
1166                 xpc_exit_sn2();
1167         else
1168                 xpc_exit_uv();
1169         return ret;
1170 }
1171
1172 module_init(xpc_init);
1173
1174 void __exit
1175 xpc_exit(void)
1176 {
1177         xpc_do_exit(xpUnloading);
1178 }
1179
1180 module_exit(xpc_exit);
1181
1182 MODULE_AUTHOR("Silicon Graphics, Inc.");
1183 MODULE_DESCRIPTION("Cross Partition Communication (XPC) support");
1184 MODULE_LICENSE("GPL");
1185
1186 module_param(xpc_hb_interval, int, 0);
1187 MODULE_PARM_DESC(xpc_hb_interval, "Number of seconds between "
1188                  "heartbeat increments.");
1189
1190 module_param(xpc_hb_check_interval, int, 0);
1191 MODULE_PARM_DESC(xpc_hb_check_interval, "Number of seconds between "
1192                  "heartbeat checks.");
1193
1194 module_param(xpc_disengage_timelimit, int, 0);
1195 MODULE_PARM_DESC(xpc_disengage_timelimit, "Number of seconds to wait "
1196                  "for disengage to complete.");
1197
1198 module_param(xpc_kdebug_ignore, int, 0);
1199 MODULE_PARM_DESC(xpc_kdebug_ignore, "Should lack of heartbeat be ignored by "
1200                  "other partitions when dropping into kdebug.");