#!/usr/bin/perl # # Comstar iSCSI Portal (itportal) # # Steffen Plotner, 2009 # # Released 2009.12.04 # # What does this tool do? # # This portal acts as an iscsi proxy to solve the problem of iscsi initiators # seeing all iscsi targets. Views managed by stmfadm still control what # luns initiators have access to. # # Why do we need this tool? # # Some initiators, once they have received all targets (SendTargets=All), will # connect to every target even if they could not reach a target because its # portal is in an IP segment it cannot talk to. The software initiator in # vmware esx does that, for example. What makes this worse is that vmware, upon # logon failure, will attempt to logon repeatedly every 2 seconds and won't stop. # The open-linux software initiator also logs onto to all discovered targets # open restart of the system, even if you manually logged onto only one target. # # How do we solve the problem? # # itportal has 3 major code paths: # # 1) It is an iscsi initiator to comstar's iscsi target port (860) to obtain the # listing of all TargetNames and their TargetAddresses. # 2) It can determine which targets an initiator should see given an iscsi iqn. # The iqn must be already configured by itadm and it must be assigned to # a host group by stmfadm. # 3) It is an iscsi target (port 3260) to frontend comstar's iscsi target in # order to handle initiator logon and SendTargets=All command - nothing else. # # When an iscsi initiator connects to port 3260, it offers its initiator name # in the logon request. We keep the initiator name and continue with the logon # sequence. When the initiator completed logging onto itportal, we expect the # SendTargets=All command from the initiator. # We then use the initiator name and query comstar for which targets the # initiator should see. # # Given the initiator name, we list the host groups and find the host group # the initiator is a member of. Note, that an initiator can only be a member # of a single host group (refer to man stmfadm). # # Enumerate all LUs and their associated views, look at each view and if the # view has a matching host group name, we track the target group name. # # For each target group name found, we enumerate the associated targets. Those # are the targets the initiator should see. # # Installation: # # To successfully run this service, I suggest you change your itadm portals to # use port 860 (it is an IANA port number for iscsi) and configure itportal to # use port 3260. That way initiators like qlogic-4010 which only support an IP # address in their gui to connect will work. Once the initiator has obtained # the list of targets, the port number associated with the target can be any. # # Shortcomings: # # * iSNS would also have worked, except that vmware esx does not support iSNS. # * itportal does not support chap. # * itportal is designed to only support environments, where a view has both # a host group and target group defined. # # Initiators tested: # # CentOS5, CentOS4, RedHat AS3, Windows 2003, Windows 2008, VMWare ESX 3, # Atto for Mac, QLA4010 # use strict; use Data::Dumper; use Getopt::Std; use IO::Socket; use POSIX qw(setsid SIGTERM); use Cwd qw(abs_path); use FindBin; use lib "$FindBin::Bin/lib"; use lib "$FindBin::Bin/etc"; use common; use config; use iscsi; # # filenames that we use # my $log_stdout = abs_path("$cfg{'log_directory'}/itportal_debug.log"); my $log_stderr = abs_path("$cfg{'log_directory'}/itportal_err.log"); my $log_access = abs_path("$cfg{'log_directory'}/itportal_access.log"); my $fh_access; my $pid_file = abs_path("$cfg{'state_directory'}/itportal.pid"); # # handle command line options # my %opts; getopts('hvdi:bfk',\%opts); if ( defined($opts{'h'}) ) { print </dev/null' or die "Can't write to /dev/null: $!"; open STDERR, '>/dev/null' or die "Can't write to /dev/null: $!"; defined(my $pid = fork) or die "Can't fork: $!"; exit if $pid; setsid or die "Can't start a new session: $!"; umask 0; # # re-open stdout and stderr # open STDOUT, ">>$log_stdout"; open STDERR, ">>$log_stderr"; # # register the pid # itportal_daemon_pid(); } #------------------------------------------------------------------------------ # iscsi portal target server #------------------------------------------------------------------------------ sub itportal_target { # # setup signals # $SIG{'CHLD'} = 'IGNORE'; $SIG{'PIPE'} = 'IGNORE'; # # listen on socket # my $server_port = $cfg{'itportal'}{'listen_port'}; logger("target> listening on port: $server_port SOMAXCONN: ".SOMAXCONN); my $server = IO::Socket::INET->new( LocalAddr => '0.0.0.0', LocalPort => $server_port, Type => SOCK_STREAM, ReuseAddr => 1, Listen => SOMAXCONN, ); if ( ! $server ) { logger_err("target_err> unable to listen on socket with $server_port"); exit(1); } # # accept connections and fork # logger("target> parent pid: $$"); while (my $client = $server->accept()) { my $fpid = fork(); if ( !defined($fpid) ) { logger_err("target_err> unable to fork"); next; } if ( $fpid == 0 ) { my $pid = $$; logger("target> child process pid: $pid"); itportal_target_session($client); exit(0); } else { logger("target> parent has launched pid: $fpid"); } } # # close listening socket # close($server); } #------------------------------------------------------------------------------ # target session #------------------------------------------------------------------------------ sub itportal_target_session { my ($client) = @_; my $ip = $client->peerhost(); logger("target> initiator from ip: $ip"); itportal_access_log($ip, "connected"); if ( !itportal_target_dowork($client) ) { logger_err("target_err> client disappeared: $ip"); } iscsi_net_disconnect($client); logger("target> initiator from ip: $ip closed"); itportal_access_log($ip, "disconnected"); } #------------------------------------------------------------------------------ # iscsi process login (no auth) and sendtargets=All request from initiator #------------------------------------------------------------------------------ sub itportal_target_dowork { my ($socket) = @_; my $ip = $socket->peerhost(); my %pdu; my $packet; # # iscsi LOGIN # logger("target> LOGIN"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("target_err> invalid packet"); return undef; } iscsi_pdu_login(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_LOGIN'} ) { logger_err("target_err> expecting Login got opcode: $pdu{'opcode'}"); return undef; } # # from the keyvalue array pull out the initiator name, alias and session # type, the rest gets returned to the client. # my $initiator_name; my $session_type; my @keyvals_out; foreach my $keyval (@{$pdu{'keyvals'}}) { if ( $keyval =~ /^InitiatorName=(\S+)$/ ) { $initiator_name = $1; logger("target> $keyval (matched)"); } elsif ( $keyval =~ /^SessionType=(\S+)$/ ) { $session_type = $1; logger("target> $keyval (matched)"); } elsif ( ! ( $keyval =~ /^InitiatorAlias=\S+$/ )) { logger("target> $keyval (keep)"); push @keyvals_out, $keyval; } } $pdu{'keyvals'} = \@keyvals_out; itportal_access_log($ip, "initiator name: $initiator_name"); if ( !defined($initiator_name) ) { logger_err("target_err> expecting InitiatorName in keyvals section"); return undef; } # # determine the current and next stage of logon negotiation # # next stage indicates how we respond and what we are listening # for. The next stage is either login operational or full phase # my $csg = $pdu{'opcode_flag'} & $iscsi_def{'ISCSI_CSG_MASK'}; my $nsg = $pdu{'opcode_flag'} & $iscsi_def{'ISCSI_NSG_MASK'}; logger("target> csg: $csg nsg: $nsg"); logger("target> nsg: SECURITY_NEGOTIATION") if ( $nsg == $iscsi_def{'ISCSI_NSG_SECURITY_NEGOTIATION'} ); logger("target> nsg: LOGINOPERATIONAL_NEGOTIATION") if ( $nsg == $iscsi_def{'ISCSI_NSG_LOGINOPERATIONAL_NEGOTIATION'} ); logger("target> nsg: FULLFEATUREPHASE") if ( $nsg == $iscsi_def{'ISCSI_NSG_FULLFEATUREPHASE'} ); # # initiator wants to negotiate logon # if ( $nsg == $iscsi_def{'ISCSI_NSG_LOGINOPERATIONAL_NEGOTIATION'} ) { # # iscsi LOGIN RESPONSE # logger("target> LOGIN RESPONSE"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_LOGIN_RESPONSE'}; $pdu{'opcode_flag'} = $iscsi_def{'ISCSI_TRANSIT_ON'} | $iscsi_def{'ISCSI_CONTINUE_OFF'} | $csg | $nsg; $pdu{'status'} = 0; $pdu{'keyvals'} = [ 'AuthMethod=None' ]; iscsi_pdu_target_sn(\%pdu); iscsi_pdu_login_response(\%pdu, \$packet); if (!iscsi_net_send($socket, \$packet)) { logger_err("target_err> unable to send to network?"); return undef; } # # iscsi LOGIN # logger("target> LOGIN"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("target_err> invalid packet"); return undef; } iscsi_pdu_login(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_LOGIN'} ) { logger_err("target_err> expecting Login got opcode: $pdu{'opcode'}"); return undef; } $csg = $pdu{'opcode_flag'} & $iscsi_def{'ISCSI_CSG_MASK'}; $nsg = $pdu{'opcode_flag'} & $iscsi_def{'ISCSI_NSG_MASK'}; logger("target> csg: $csg nsg: $nsg"); logger("target> nsg: SECURITY_NEGOTIATION") if ( $nsg == $iscsi_def{'ISCSI_NSG_SECURITY_NEGOTIATION'} ); logger("target> nsg: LOGINOPERATIONAL_NEGOTIATION") if ( $nsg == $iscsi_def{'ISCSI_NSG_LOGINOPERATIONAL_NEGOTIATION'} ); logger("target> nsg: FULLFEATUREPHASE") if ( $nsg == $iscsi_def{'ISCSI_NSG_FULLFEATUREPHASE'} ); } # # iscsi LOGIN RESPONSE # logger("target> LOGIN RESPONSE"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_LOGIN_RESPONSE'}; $pdu{'opcode_flag'} = $iscsi_def{'ISCSI_TRANSIT_ON'} | $iscsi_def{'ISCSI_CONTINUE_OFF'} | $csg | $nsg; $pdu{'status'} = 0; iscsi_pdu_target_sn(\%pdu); iscsi_pdu_login_response(\%pdu, \$packet); if (!iscsi_net_send($socket, \$packet)) { logger_err("target_err> unable to send to network?"); return undef; } # # iscsi TEXT COMMAND/LOGOUT # logger("target> TEXT COMMAND"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("target_err> invalid packet"); return undef; } iscsi_pdu_text_command(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_TEXT_COMMAND'} ) { logger_err("target_err> expecting Text Command got opcode: $pdu{'opcode'}"); return undef; } # # check for SendTargets=All # my $found = 0; foreach my $keyval (@{$pdu{'keyvals'}}) { if ( $keyval =~ /SendTargets=All/ ) { itportal_access_log($ip, "initiator request: $keyval"); $found = 1; } } if ( !$found ) { logger_err("target_err> expecting SendTargets=All"); return undef; } # # based on the initiator name, determine the related targets # my $targets_related = stmf_query_targets($initiator_name); # # get all targets available # my $target_ip = $cfg{'itportal'}{'target_ip'}; my $target_port = $cfg{'itportal'}{'target_port'}; my $targets_all = itportal_initiator_sendtargets_all($initiator_name, $target_ip, $target_port); # # enumerate over the related targets and determine from the all # targets the maching ones and include the following TargetAddress # $pdu{'keyvals'} = undef; foreach my $target_related (@{$targets_related}) { my $match = 0; foreach my $target (@{$targets_all} ) { if ( $target =~ /^TargetName=(\S+)$/ ) { my $target_name = $1; if ( $target_name eq $target_related ) { $match = 1; push @{$pdu{'keyvals'}}, $target; logger("target> allowing $target"); } else { $match = 0; } } if ( $target =~ /^TargetAddress=\S+$/ && $match ) { push @{$pdu{'keyvals'}}, $target; logger("target> allowing $target"); } } } foreach my $keyval (@{$pdu{'keyvals'}}) { itportal_access_log($ip, "target response: $keyval"); } # # iscsi TEXT RESPONSE # $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_TEXT_RESPONSE'}; iscsi_pdu_target_sn(\%pdu); iscsi_pdu_text_response(\%pdu, \$packet); if (!iscsi_net_send($socket, \$packet)) { logger_err("target_err> unable to send to network?"); return undef; } # # iscsi LOGOUT REQUEST # logger("target> LOGOUT REQUEST"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger("target> initiator did not send logout request, closing connection"); return 1; } iscsi_pdu_logout_request(\%pdu, $packet); if ( $pdu{'opcode'} == $iscsi_def{'OPCODE_LOGOUT_REQUEST'} ) { # # iscsi LOGOUT RESPONSE # logger("target> initiator is requesting to logout... bye"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_LOGOUT_RESPONSE'}; iscsi_pdu_target_sn(\%pdu); iscsi_pdu_logout_response(\%pdu, \$packet); if (!iscsi_net_send($socket, \$packet)) { logger_err("target_err> unable to send to network?"); return undef; } return 1; } logger_err("target_err> expecting Logout Request got opcode: $pdu{'opcode'}"); return undef; } #------------------------------------------------------------------------------ # iscsi initiator SendTargets=All # # let the proxy perform a SendTargets=All command and return all targets # \[ # 'TargetName=iqn.1990-01.edu.amherst.zfs-san0:tgt00.ip1.vlan141.spio0', # 'TargetAddress=148.85.141.70:3260,2', # 'TargetName=iqn.1990-01.edu.amherst.zfs-san0:tgt01.ip1.vlan141.spio1', # 'TargetAddress=148.85.141.71:3260,2', # 'TargetName=iqn.1990-01.edu.amherst.zfs-san0:tgt02.ip2.vlan141.mpio0', # 'TargetAddress=148.85.141.72:3260,2', # 'TargetAddress=148.85.141.73:3260,2', # 'TargetName=iqn.1990-01.edu.amherst.zfs-san0:tgt03.ip1.vlan141.mpio1', # 'TargetAddress=148.85.141.74:3260,2', # 'TargetName=iqn.1990-01.edu.amherst.zfs-san0:tgt04.ip1.vlan141.mpio1', # 'TargetAddress=148.85.141.75:3260,2' # ]; #------------------------------------------------------------------------------ sub itportal_initiator_sendtargets_all { my ($initiator_name, $target_ip, $target_port) = @_; logger("initiator> initiator_name: $initiator_name"); my %pdu; my $packet; # # iscsi connect # my $socket = iscsi_net_connect($target_ip, $target_port); if ( !$socket ) { logger_err("initiator_err> unable to connect with ip: $target_ip port: $target_port"); return undef; } # # iscsi LOGIN # logger("initiator> LOGIN"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_LOGIN'}; $pdu{'opcode_flag'} = $iscsi_def{'ISCSI_TRANSIT_ON'} | $iscsi_def{'ISCSI_CONTINUE_OFF'} | $iscsi_def{'ISCSI_CSG_LOGINOPERATIONAL_NEGOTIATION'} | $iscsi_def{'ISCSI_NSG_FULLFEATUREPHASE'}; $pdu{'keyvals'} = [ "InitiatorName=$initiator_name", "SessionType=Discovery", "MaxRecvDataSegmentLength=$cfg{'itportal'}{'max_recv_data_segment_length'}", ]; iscsi_pdu_login(\%pdu, \$packet); iscsi_pdu_initiator_sn(\%pdu); if (!iscsi_net_send($socket, \$packet)) { logger_err("initiator_err> unable to send to network?"); return undef; } # # iscsi LOGIN RESPONSE # logger("initiator> LOGIN RESPONSE"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("initiator_err> invalid packet"); return undef; } iscsi_pdu_login_response(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_LOGIN_RESPONSE'} || $pdu{'status'} != 0 ) { logger_err("initiator_err> target logon error opcode: $pdu{'opcode'} status: $pdu{'status'}"); return undef; } # # iscsi TEXT COMMAND # logger("initiator> TEXT COMMAND"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_TEXT_COMMAND'}; $pdu{'opcode_flag'} = $iscsi_def{'ISCSI_TRANSIT_ON'} | $iscsi_def{'ISCSI_CONTINUE_OFF'}; $pdu{'keyvals'} = [ "SendTargets=All", ]; iscsi_pdu_text_command(\%pdu, \$packet); iscsi_pdu_initiator_sn(\%pdu); if (!iscsi_net_send($socket, \$packet)) { logger_err("initiator_err> unable to send to network?"); return undef; } # # iscsi TEXT RESPONSE # logger("initiator> TEXT RESPONSE"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("initiator_err> invalid packet"); return undef; } iscsi_pdu_text_response(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_TEXT_RESPONSE'} ) { logger_err("initiator_err> target Text Response error opcode: $pdu{'opcode'}"); return undef; } # # keep the targets # my $targets = $pdu{'keyvals'}; $pdu{'keyvals'} = undef; # # iscsi LOGOUT REQUEST # logger("initiator> LOGOUT REQUEST"); $pdu{'cmd'} = $iscsi_def{'CMD_SEND'}; $pdu{'opcode'} = $iscsi_def{'OPCODE_LOGOUT_REQUEST'}; iscsi_pdu_logout_request(\%pdu, \$packet); iscsi_pdu_initiator_sn(\%pdu); if (!iscsi_net_send($socket, \$packet)) { logger_err("initiator_err> unable to send to network?"); return undef; } # # iscsi LOGOUT RESPONSE # logger("initiator> LOGOUT RESPONSE"); $pdu{'cmd'} = $iscsi_def{'CMD_RECV'}; $packet = iscsi_net_recv($socket); if ( !defined($packet) ) { logger_err("initiator_err> invalid packet"); return undef; } iscsi_pdu_logout_response(\%pdu, $packet); if ( $pdu{'opcode'} != $iscsi_def{'OPCODE_LOGOUT_RESPONSE'} ) { logger_err("initiator_err> Logout Response expected, got opcode: $pdu{'opcode'}"); return undef; } # # iscsi disconnect # iscsi_net_disconnect($socket); return $targets; } #------------------------------------------------------------------------------ # using stmfadm commands determine which targets should be exposed given # a initiator name #------------------------------------------------------------------------------ sub stmf_query_targets { my ($initiator_name) = @_; my $cmd; my $data; # # stmfadm list-hg to retrieve the host group name of the initiator. An # initiator can only be a member of a single host group (man stmfadm) # $cmd = "stmfadm list-hg -v"; logger("stmf> cmd: $cmd"); $data = `$cmd`; my $host_group; foreach my $line (split("\n", $data) ) { logger("stmf> $line"); if ( $line =~ /Host Group: (\S+)/ ) { $host_group = $1; } if ( $line =~ /Member: (\S+)/ && $1 eq $initiator_name ) { last; } } logger("stmf> host_group: $host_group initiator_name: $initiator_name"); if ( ! defined($host_group) ) { logger_err("stmf_err> unable to find host_group for initiator $initiator_name"); return undef; } # # lookup all LUs # $cmd = "stmfadm list-lu"; logger("stmf> cmd: $cmd"); $data = `$cmd`; my @LUs; foreach my $line (split("\n", $data) ) { if ( $line =~ /LU Name: (\S+)/ ) { my $LU = $1; logger("stmf> LU: $LU"); push @LUs, $LU; } } # # foreach given LU, lookup the views and for a view with a matching # host group, remember the target group (cumulatively) # my @target_groups; foreach my $LU (@LUs) { $cmd = "stmfadm list-view -l $LU"; logger("stmf> cmd: $cmd"); $data = `$cmd`; my $matched_hg; foreach my $line (split("\n", $data) ) { logger("stmf> $line"); if ( $line =~ /Host group : (\S+)/ ) { my $hg = $1; $matched_hg = 0; if ( $hg eq $host_group ) { logger("stmf> found matching host group, adding next target group"); $matched_hg = 1; } } if ( $line =~ /Target group : (\S+)/ && $matched_hg ) { my $tg = $1; push @target_groups, $tg; logger("stmf> adding target group: $tg"); } } } # # given the target_group, enumerate them and obtain from each of those # the target ports (iqn). A target port can only be a member of # a single target group (man stmfadm). # my @targets_related; foreach my $target_group (@target_groups) { $cmd = "stmfadm list-tg -v $target_group"; logger("stmf> cmd: $cmd"); $data = `$cmd`; foreach my $line (split("\n", $data) ) { logger("stmf> $line"); if ( $line =~ /Member: (\S+)/ ) { my $member = $1; logger("stmf> found related target port: $member"); push @targets_related, $member; } } } return \@targets_related; } #------------------------------------------------------------------------------ # itportal access logger #------------------------------------------------------------------------------ sub itportal_access_open { open $fh_access, ">>$log_access"; } sub itportal_access_log { my ($ip, $msg) = @_; my $line = date_time_string()." $ip $msg"; print $fh_access "$line\n"; } sub itportal_access_close { close $fh_access; } #------------------------------------------------------------------------------ # itportal track pid #------------------------------------------------------------------------------ sub itportal_daemon_pid { my $pid = $$; open (FH, ">$pid_file") or die "Unable to open pid file: $pid_file reason: $!"; print FH "$pid"; close FH; } sub itportal_daemon_kill { open (FH, "<$pid_file") or die "Unable to open pid file: $pid_file reason: $!"; my $pid = ; kill SIGTERM, $pid; close FH; }